diff --git a/metacat-client/src/main/java/com/netflix/metacat/client/Client.java b/metacat-client/src/main/java/com/netflix/metacat/client/Client.java index fd224844a..fecdac692 100644 --- a/metacat-client/src/main/java/com/netflix/metacat/client/Client.java +++ b/metacat-client/src/main/java/com/netflix/metacat/client/Client.java @@ -21,15 +21,16 @@ import com.fasterxml.jackson.datatype.guava.GuavaModule; import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule; import com.google.common.base.Preconditions; +import com.netflix.metacat.client.api.MetacatV1; +import com.netflix.metacat.client.api.MetadataV1; +import com.netflix.metacat.client.api.ParentChildRelV1; +import com.netflix.metacat.client.api.PartitionV1; +import com.netflix.metacat.client.api.ResolverV1; import com.netflix.metacat.client.api.TagV1; import com.netflix.metacat.client.module.JacksonDecoder; import com.netflix.metacat.client.module.JacksonEncoder; import com.netflix.metacat.client.module.MetacatErrorDecoder; import com.netflix.metacat.common.MetacatRequestContext; -import com.netflix.metacat.client.api.MetacatV1; -import com.netflix.metacat.client.api.MetadataV1; -import com.netflix.metacat.client.api.PartitionV1; -import com.netflix.metacat.client.api.ResolverV1; import com.netflix.metacat.common.json.MetacatJsonLocator; import feign.Feign; import feign.Request; @@ -58,6 +59,8 @@ public final class Client { private final ResolverV1 resolverApi; private final TagV1 tagApi; + private final ParentChildRelV1 parentChildRelApi; + private Client( final String host, final feign.Client client, @@ -93,6 +96,7 @@ private Client( metadataApi = getApiClient(MetadataV1.class); resolverApi = getApiClient(ResolverV1.class); tagApi = getApiClient(TagV1.class); + parentChildRelApi = getApiClient(ParentChildRelV1.class); } /** @@ -163,6 +167,15 @@ public TagV1 getTagApi() { return tagApi; } + /** + * Return an API instance that can be used to interact with + * the metacat server for parent child relationship metadata. + * @return An instance api conforming to ParentChildRelV1 interface + */ + public ParentChildRelV1 getParentChildRelApi() { + return parentChildRelApi; + } + /** * Builder class to build the metacat client. */ diff --git a/metacat-client/src/main/java/com/netflix/metacat/client/api/ParentChildRelV1.java b/metacat-client/src/main/java/com/netflix/metacat/client/api/ParentChildRelV1.java new file mode 100644 index 000000000..546030f69 --- /dev/null +++ b/metacat-client/src/main/java/com/netflix/metacat/client/api/ParentChildRelV1.java @@ -0,0 +1,43 @@ +package com.netflix.metacat.client.api; + +import javax.ws.rs.Path; +import javax.ws.rs.Consumes; +import javax.ws.rs.Produces; +import javax.ws.rs.GET; +import javax.ws.rs.PathParam; + +import javax.ws.rs.core.MediaType; +import com.netflix.metacat.common.dto.notifications.ChildInfoDto; + +import java.util.Set; + +/** + * Metacat API for managing parent child relation. + * + * @author Yingjianw + */ + +@Path("/mds/v1/parentChildRel") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +public interface ParentChildRelV1 { + /** + * Return the list of children. + * @param catalogName catalogName + * @param databaseName databaseName + * @param tableName tableName + * @return list of childInfos + */ + @GET + @Path("children/catalog/{catalog-name}/database/{database-name}/table/{table-name}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + Set getChildren( + @PathParam("catalog-name") + String catalogName, + @PathParam("database-name") + String databaseName, + @PathParam("table-name") + String tableName + ); +} diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/converter/ConverterUtil.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/converter/ConverterUtil.java index 05690c312..5ad54ace8 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/converter/ConverterUtil.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/converter/ConverterUtil.java @@ -33,6 +33,7 @@ import com.netflix.metacat.common.dto.Sort; import com.netflix.metacat.common.dto.StorageDto; import com.netflix.metacat.common.dto.TableDto; +import com.netflix.metacat.common.dto.notifications.ChildInfoDto; import com.netflix.metacat.common.server.connectors.ConnectorRequestContext; import com.netflix.metacat.common.server.connectors.model.AuditInfo; import com.netflix.metacat.common.server.connectors.model.CatalogInfo; @@ -46,6 +47,7 @@ import com.netflix.metacat.common.server.connectors.model.PartitionsSaveResponse; import com.netflix.metacat.common.server.connectors.model.StorageInfo; import com.netflix.metacat.common.server.connectors.model.TableInfo; +import com.netflix.metacat.common.server.model.ChildInfo; import lombok.NonNull; import org.dozer.CustomConverter; import org.dozer.DozerBeanMapper; @@ -276,5 +278,13 @@ public PartitionsSaveResponseDto toPartitionsSaveResponseDto(final PartitionsSav return mapper.map(partitionsSaveResponse, PartitionsSaveResponseDto.class); } - + /** + * Convert ChildInfo to ChildInfoDto. + * + * @param childInfo childInfo + * @return childInfo dto + */ + public ChildInfoDto toChildInfoDto(final ChildInfo childInfo) { + return mapper.map(childInfo, ChildInfoDto.class); + } } diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/BaseRelEntityInfo.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/BaseRelEntityInfo.java new file mode 100644 index 000000000..41a4b940c --- /dev/null +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/BaseRelEntityInfo.java @@ -0,0 +1,21 @@ +package com.netflix.metacat.common.server.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; + +/** + * Base class to represent relation entity. + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public abstract class BaseRelEntityInfo implements Serializable { + private static final long serialVersionUID = 9121109874202888889L; + private String name; + private String relationType; + private String uuid; + +} diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/ChildInfo.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/ChildInfo.java new file mode 100644 index 000000000..7deb0945d --- /dev/null +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/ChildInfo.java @@ -0,0 +1,23 @@ +package com.netflix.metacat.common.server.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * ChildInfo. + */ +@EqualsAndHashCode(callSuper = true) +@AllArgsConstructor +@Data +public class ChildInfo extends BaseRelEntityInfo { + /** + Constructor with all params. + @param name name of the entity + @param relationType type of the relation + @param uuid uuid of the entity + */ + public ChildInfo(final String name, final String relationType, final String uuid) { + super(name, relationType, uuid); + } +} diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/ParentInfo.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/ParentInfo.java new file mode 100644 index 000000000..ff0cca8b4 --- /dev/null +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/ParentInfo.java @@ -0,0 +1,29 @@ +package com.netflix.metacat.common.server.model; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * ParentInfo. + */ +@EqualsAndHashCode(callSuper = true) +@Data +public class ParentInfo extends BaseRelEntityInfo { + + /** + Empty Constructor. + */ + public ParentInfo() { + + } + + /** + Constructor with all params. + @param name name of the entity + @param relationType type of the relation + @param uuid uuid of the entity + */ + public ParentInfo(final String name, final String relationType, final String uuid) { + super(name, relationType, uuid); + } +} diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataConstants.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataConstants.java new file mode 100644 index 000000000..f2f7dd666 --- /dev/null +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataConstants.java @@ -0,0 +1,55 @@ +package com.netflix.metacat.common.server.usermetadata; + +/** + * ParentChildRelMetadataConstants. + * + * @author yingjianw + */ +public final class ParentChildRelMetadataConstants { + /** + * During get and create, top level key specified in DefinitionMetadata that indicates the parent child infos. + */ + public static final String PARENT_CHILD_RELINFO = "parentChildRelationInfo"; + /** + * During create, nested level key specified in DefinitionMetadata['parentChildRelationInfo'] + * that indicate the parent table name. + */ + public static final String PARENT_NAME = "root_table_name"; + /** + * During create, nested level key specified in DefinitionMetadata['parentChildRelationInfo'] + * that indicates the parent table uuid. + */ + public static final String PARENT_UUID = "root_table_uuid"; + /** + * During create, nested level key specified in DefinitionMetadata['parentChildRelationInfo'] + * that indicates the child table uuid. + */ + public static final String CHILD_UUID = "child_table_uuid"; + + /** + * During create, nested level key specified in DefinitionMetadata['parentChildRelationInfo'] + * that indicates relationType. + */ + public static final String RELATION_TYPE = "relationType"; + + /** + * During get, the nested key specified in DefinitionMetadata[PARENTCHILDRELINFO] that indicates parent infos. + */ + public static final String PARENT_INFOS = "parentInfos"; + + /** + * During get, the nested key specified in DefinitionMetadata[PARENTCHILDRELINFO] that indicates child infos. + */ + public static final String CHILD_INFOS = "childInfos"; + + /** + * During get, the nested key specified in DefinitionMetadata[PARENTCHILDRELINFO] + * that indicates if a table is parent. + */ + public static final String IS_PARENT = "isParent"; + + private ParentChildRelMetadataConstants() { + + } + +} diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataService.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataService.java new file mode 100644 index 000000000..b8e6220a2 --- /dev/null +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataService.java @@ -0,0 +1,106 @@ +package com.netflix.metacat.common.server.usermetadata; +import com.netflix.metacat.common.QualifiedName; +import com.netflix.metacat.common.dto.notifications.ChildInfoDto; +import com.netflix.metacat.common.server.model.ChildInfo; +import com.netflix.metacat.common.server.model.ParentInfo; + +import java.util.Set; + +/** + * Parent-Child Relationship Metadata Service API. + * + * @author yingjianw + */ +public interface ParentChildRelMetadataService { + /** + * Establishes a parent-child relationship with a specified relation type. + * Currently, exceptions are thrown in the following cases: + * 1. Attempting to create a child table as the parent of another child table. + * 2. Attempting to create a parent table on top of a parent table + * 3. A child table having more than one parent. + * + * @param parentName the name of the parent entity + * @param parentUUID the uuid of the parent + * @param childName the name of the child entity + * @param childUUID the uuid of the child + * @param relationType the type of the relationship + */ + void createParentChildRelation( + QualifiedName parentName, + String parentUUID, + QualifiedName childName, + String childUUID, + String relationType + ); + + /** + * Deletes a parent-child relationship with a specified relation type. + * This function is only called in the recovery process when + * we first create the parent-child relationship but fail to create the table. + * + * @param parentName the name of the parent entity + * @param parentUUID the uuid of the parent + * @param childName the name of the child entity + * @param childUUID the uuid of the child + * @param type the type of the relationship + */ + void deleteParentChildRelation( + QualifiedName parentName, + String parentUUID, + QualifiedName childName, + String childUUID, + String type + ); + + /** + * Renames `oldName` to `newName` in the parentChildRelationship store. + * This involves two steps: + * 1. Rename all records where the child is `oldName` to `newName` + * 2. Rename all records where the parent is `oldName` to `newName` + * + * @param oldName the current name to be renamed + * @param newName the new name to rename to + */ + void rename( + QualifiedName oldName, + QualifiedName newName + ); + + /** + * Removes the entity from the parentChildRelationship store. + * This involves two steps: + * 1. drop all records where the child column = `name` + * 2. drop all records where the parent column = `name` + * @param name the name of the entity to drop + */ + void drop( + QualifiedName name + ); + + /** + * get the set of parent for the input name. + * @param name name + * @return parentInfo + */ + Set getParents( + QualifiedName name + ); + + /** + * get the set of children for the input name. + * @param name name + * @return a set of ChildInfo + */ + Set getChildren( + QualifiedName name + ); + + /** + * get the set of children dto for the input name. + * @param name name + * @return a set of ChildInfo + */ + Set getChildrenDto( + QualifiedName name + ); +} diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelServiceException.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelServiceException.java new file mode 100644 index 000000000..124a9c588 --- /dev/null +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelServiceException.java @@ -0,0 +1,25 @@ +package com.netflix.metacat.common.server.usermetadata; + +/** + * Parent Child Rel Service exception. + */ +public class ParentChildRelServiceException extends RuntimeException { + /** + * Constructor. + * + * @param m message + */ + public ParentChildRelServiceException(final String m) { + super(m); + } + + /** + * Constructor. + * + * @param m message + * @param e exception + */ + public ParentChildRelServiceException(final String m, final Exception e) { + super(m, e); + } +} diff --git a/metacat-common/src/main/java/com/netflix/metacat/common/dto/notifications/ChildInfoDto.java b/metacat-common/src/main/java/com/netflix/metacat/common/dto/notifications/ChildInfoDto.java new file mode 100644 index 000000000..bf150c4af --- /dev/null +++ b/metacat-common/src/main/java/com/netflix/metacat/common/dto/notifications/ChildInfoDto.java @@ -0,0 +1,31 @@ +package com.netflix.metacat.common.dto.notifications; + +import com.netflix.metacat.common.dto.BaseDto; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +/** + * ChildInfo information. + */ +@SuppressWarnings("unused") +@Data +@EqualsAndHashCode(callSuper = false) +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class ChildInfoDto extends BaseDto { + private static final long serialVersionUID = 9121109874202088789L; + /* Name of the child */ + @ApiModelProperty(value = "name of the child") + private String name; + /* Type of the relation */ + @ApiModelProperty(value = "type of the relation") + private String relationType; + /* uuid of the table */ + @ApiModelProperty(value = "uuid of the table") + private String uuid; +} diff --git a/metacat-functional-tests/metacat-test-cluster/datastores/mysql/docker-entrypoint-initdb.d/metacat.sql b/metacat-functional-tests/metacat-test-cluster/datastores/mysql/docker-entrypoint-initdb.d/metacat.sql index 189700dc6..5a58eb4ef 100644 --- a/metacat-functional-tests/metacat-test-cluster/datastores/mysql/docker-entrypoint-initdb.d/metacat.sql +++ b/metacat-functional-tests/metacat-test-cluster/datastores/mysql/docker-entrypoint-initdb.d/metacat.sql @@ -44,6 +44,21 @@ CREATE TABLE data_metadata_delete ( KEY date_created (date_created) ) DEFAULT CHARSET=latin1; +-- +-- Table structure for table `parent_child_relation` +-- +DROP TABLE IF EXISTS `parent_child_relation`; +CREATE TABLE `parent_child_relation` ( + `parent` varchar(255) NOT NULL, + `parent_uuid` varchar(255) NOT NULL, + `child` varchar(255) NOT NULL, + `child_uuid` varchar(255) NOT NULL, + `relation_type` varchar(255) NOT NULL, + PRIMARY KEY (`parent`, `child`, `parent_uuid`, `child_uuid`, `relation_type`), + INDEX `idx_child` (`child`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + -- -- Table structure for table `definition_metadata` -- diff --git a/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy index 697d8d647..9ccfd1e70 100644 --- a/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy +++ b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy @@ -15,13 +15,16 @@ */ package com.netflix.metacat +import com.fasterxml.jackson.databind.ObjectMapper import com.netflix.metacat.client.Client import com.netflix.metacat.client.api.MetacatV1 import com.netflix.metacat.client.api.MetadataV1 +import com.netflix.metacat.client.api.ParentChildRelV1 import com.netflix.metacat.client.api.PartitionV1 import com.netflix.metacat.client.api.TagV1 import com.netflix.metacat.common.QualifiedName import com.netflix.metacat.common.dto.* +import com.netflix.metacat.common.dto.notifications.ChildInfoDto import com.netflix.metacat.common.exception.MetacatAlreadyExistsException import com.netflix.metacat.common.exception.MetacatBadRequestException import com.netflix.metacat.common.exception.MetacatNotFoundException @@ -31,6 +34,7 @@ import com.netflix.metacat.common.exception.MetacatUnAuthorizedException import com.netflix.metacat.common.json.MetacatJson import com.netflix.metacat.common.json.MetacatJsonLocator import com.netflix.metacat.common.server.connectors.exception.InvalidMetaException +import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataConstants import com.netflix.metacat.connector.hive.util.PartitionUtil import com.netflix.metacat.testdata.provider.PigDataDtoProvider import feign.Logger @@ -59,6 +63,7 @@ class MetacatSmokeSpec extends Specification { public static PartitionV1 partitionApi public static MetadataV1 metadataApi public static TagV1 tagApi + public static ParentChildRelV1 parentChildRelV1 public static MetacatJson metacatJson = new MetacatJsonLocator() def setupSpec() { @@ -85,6 +90,7 @@ class MetacatSmokeSpec extends Specification { partitionApi = client.partitionApi tagApi = client.tagApi metadataApi = client.metadataApi + parentChildRelV1 = client.parentChildRelApi } @Shared @@ -137,6 +143,19 @@ class MetacatSmokeSpec extends Specification { } } + static void initializeParentChildRelDefinitionMetadata(TableDto tableDto, + String parent, + String parent_uuid, + String child_uuid) { + def mapper = new ObjectMapper() + def innerNode = mapper.createObjectNode() + innerNode.put(ParentChildRelMetadataConstants.PARENT_NAME, parent) + innerNode.put(ParentChildRelMetadataConstants.PARENT_UUID, parent_uuid) + innerNode.put(ParentChildRelMetadataConstants.CHILD_UUID, child_uuid) + + tableDto.definitionMetadata.put(ParentChildRelMetadataConstants.PARENT_CHILD_RELINFO, innerNode) + } + def createAllTypesTable() { when: createTable('embedded-hive-metastore', 'smoke_db', 'metacat_all_types') @@ -1962,4 +1981,423 @@ class MetacatSmokeSpec extends Specification { api.deleteTable(catalogName, databaseName, "table2") api.deleteTable(catalogName, databaseName, "table3") } + + def 'testCloneTableE2E'() { + given: + def catalogName = 'embedded-fast-hive-metastore' + def databaseName = 'iceberg_db' + + // First parent child connected component + def parent1 = "parent1" + def parent1UUID = "p1_uuid" + def renameParent1 = "rename_parent1" + def parent1Uri = isLocalEnv ? String.format('file:/tmp/%s/%s', databaseName, parent1) : null + def parent1FullName = catalogName + "/" + databaseName + "/" + parent1 + + def child11 = "child11" + def child11UUID = "c11_uuid" + def renameChild11 = "rename_child11" + def child11Uri = isLocalEnv ? String.format('file:/tmp/%s/%s', databaseName, child11) : null + def child11FullName = catalogName + "/" + databaseName + "/" + child11 + + def child12 = "child12" + def child12UUID = "c12_uuid" + def child12Uri = isLocalEnv ? String.format('file:/tmp/%s/%s', databaseName, child12) : null + + def grandChild121 = "grandchild121" + def grandChild121UUID= "gc121_uuid" + + def grandParent1 = "grandparent1" + def grandParent1FullName = catalogName + "/" + databaseName + "/" + grandParent1 + def grandParent1UUID = "grandparent2_uuid" + def grantParent1Uri = isLocalEnv ? String.format('file:/tmp/%s/%s', databaseName, parent1) : null + + + // Second parent child connected component + def parent2 = "parent2" + def parent2UUID = "p2_uuid" + def parent2Uri = isLocalEnv ? String.format('file:/tmp/%s/%s', databaseName, parent2) : null + def parent2FullName = catalogName + "/" + databaseName + "/" + parent2 + def child21 = "child21" + def child21UUID = "c21_uuid" + def child21Uri = isLocalEnv ? String.format('file:/tmp/%s/%s', databaseName, child21) : null + + try { + api.createDatabase(catalogName, databaseName, new DatabaseCreateRequestDto()) + } catch (Exception ignored) { + } + + /* + Step 1: Create one Parent (parent1) and one Child (child11) + */ + when: + // Create Parent1 + def parent1TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, parent1, 'amajumdar', parent1Uri) + api.createTable(catalogName, databaseName, parent1, parent1TableDto) + + // Create child11 Table with parent = parent1 + def child11TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, child11, 'amajumdar', child11Uri) + initializeParentChildRelDefinitionMetadata(child11TableDto, parent1FullName, parent1UUID, child11UUID) + child11TableDto.definitionMetadata.put("random_key", "random_value") + api.createTable(catalogName, databaseName, child11, child11TableDto) + + def parent1Table = api.getTable(catalogName, databaseName, parent1, true, true, false) + def child11Table = api.getTable(catalogName, databaseName, child11, true, true, false) + def child11ParentChildRelationInfo = child11Table.definitionMetadata.get("parentChildRelationInfo") + then: + // Test Parent 1 parentChildInfo + assert parent1Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() + assert parentChildRelV1.getChildren(catalogName, databaseName, parent1) == [new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child11", "CLONE", "c11_uuid")] as Set + + // Test Child11 parentChildInfo + assert !child11Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + assert child11Table.definitionMetadata.get("random_key").asText() == "random_value" + JSONAssert.assertEquals(child11ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent1","relationType":"CLONE", "uuid":"p1_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child11).isEmpty() + + /* + Step 2: create another table with the same child1 name but different uuid under the same parent should fail + */ + when: + child11TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, child11, 'amajumdar', child11Uri) + initializeParentChildRelDefinitionMetadata(child11TableDto, parent1FullName, parent1UUID, "random_uuid") + api.createTable(catalogName, databaseName, child11, child11TableDto) + then: + def e = thrown(Exception) + assert e.message.contains("Cannot have a child table having more than one parent") + + /* + Step 3: create another table with the same name different uuid without specifying the parent child relation + but should fail because the table already exists + This test the revert should not impact the previous record + */ + when: + child11TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, child11, 'amajumdar', child11Uri) + api.createTable(catalogName, databaseName, child11, child11TableDto) + then: + e = thrown(Exception) + assert e.message.contains("already exists") + + // Test Parent 1 parentChildInfo + assert parent1Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() + assert parentChildRelV1.getChildren(catalogName, databaseName, parent1) == [new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child11", "CLONE", "c11_uuid")] as Set + + // Test Child11 parentChildInfo + assert !child11Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + assert child11Table.definitionMetadata.get("random_key").asText() == "random_value" + JSONAssert.assertEquals(child11ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent1","relationType":"CLONE", "uuid":"p1_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child11).isEmpty() + + + /* + Step 4: Create a second child (child12) pointing to parent = parent1 + */ + when: + // Create Child2 Table + def child12TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, child12, 'amajumdar', child12Uri) + initializeParentChildRelDefinitionMetadata(child12TableDto, parent1FullName, parent1UUID, child12UUID) + api.createTable(catalogName, databaseName, child12, child12TableDto) + parent1Table = api.getTable(catalogName, databaseName, parent1, true, true, false) + def child12Table = api.getTable(catalogName, databaseName, child12, true, true, false) + def child12ParentChildRelationInfo = child12Table.definitionMetadata.get("parentChildRelationInfo") + + then: + // Test Parent 1 parentChildInfo + assert parent1Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() + assert parentChildRelV1.getChildren(catalogName, databaseName, parent1) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child11", "CLONE", "c11_uuid"), + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid") + ] as Set + + // Test Child12 parentChildInfo + assert !child12Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + JSONAssert.assertEquals(child12ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child12).isEmpty() + + /* + Step 5: create a parent table on top of another parent table should fail + */ + when: + def grandParent1TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, grandParent1, 'amajumdar', grantParent1Uri) + api.createTable(catalogName, databaseName, grandParent1, grandParent1TableDto) + + def parent1TableDtoCopy = PigDataDtoProvider.getTable(catalogName, databaseName, parent1, 'amajumdar', parent1Uri) + initializeParentChildRelDefinitionMetadata(parent1TableDtoCopy, grandParent1FullName, grandParent1UUID, parent1UUID) + api.createTable(catalogName, databaseName, parent1, parent1TableDtoCopy) + then: + e = thrown(Exception) + assert e.message.contains("Cannot create a parent table on top of another parent") + + /* + Step 6: create another table with the same parent1 name but should fail because the table already exists + Test the revert should not impact the previous record + */ + when: + api.createTable(catalogName, databaseName, parent1, parent1TableDto) + then: + e = thrown(Exception) + assert e.message.contains("already exists") + + // Test Parent 1 parentChildInfo + assert parent1Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() + assert parentChildRelV1.getChildren(catalogName, databaseName, parent1) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child11", "CLONE", "c11_uuid"), + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid") + ] as Set + + // Test Child11 parentChildInfo + assert !child11Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + assert child11Table.definitionMetadata.get("random_key").asText() == "random_value" + JSONAssert.assertEquals(child11ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent1","relationType":"CLONE", "uuid":"p1_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child11).isEmpty() + + /* + Step 7: Create one grandChild As a Parent of A child table should fail + */ + when: + def grandchild121TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, grandChild121, 'amajumdar', null) + initializeParentChildRelDefinitionMetadata(grandchild121TableDto, child11FullName, child11UUID, grandChild121UUID) + + api.createTable(catalogName, databaseName, grandChild121, grandchild121TableDto) + assert parentChildRelV1.getChildren(catalogName, databaseName, grandChild121).isEmpty() + + then: + e = thrown(Exception) + assert e.message.contains("Cannot create a child table as parent") + + /* + Step 8: Create another parent child that is disconnected with the above + */ + when: + // Create Parent2 + def parent2TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, parent2, 'amajumdar', parent2Uri) + api.createTable(catalogName, databaseName, parent2, parent2TableDto) + + // Create child21 Table with parent = parent2 + def child21TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, child21, 'amajumdar', child21Uri) + initializeParentChildRelDefinitionMetadata(child21TableDto, parent2FullName, parent2UUID, child21UUID) + api.createTable(catalogName, databaseName, child21, child21TableDto) + def parent2Table = api.getTable(catalogName, databaseName, parent2, true, true, false) + def child21Table = api.getTable(catalogName, databaseName, child21, true, true, false) + def child21ParentChildRelationInfo = child21Table.definitionMetadata.get("parentChildRelationInfo") + + then: + // Test Parent 2 parentChildInfo + assert parent2Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() + assert parentChildRelV1.getChildren(catalogName, databaseName, parent2) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child21", "CLONE", "c21_uuid") + ] as Set + + // Test Child21 parentChildInfo + JSONAssert.assertEquals(child21ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent2","relationType":"CLONE","uuid":"p2_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child21).isEmpty() + + /* + Step 9: Rename parent1 to newParent1 + */ + when: + api.renameTable(catalogName, databaseName, parent1, renameParent1) + parent1Table = api.getTable(catalogName, databaseName, renameParent1, true, true, false) + child11Table = api.getTable(catalogName, databaseName, child11, true, true, false) + child11ParentChildRelationInfo = child11Table.definitionMetadata.get("parentChildRelationInfo") + child12Table = api.getTable(catalogName, databaseName, child12, true, true, false) + child12ParentChildRelationInfo = child12Table.definitionMetadata.get("parentChildRelationInfo") + + then: + // Test Parent 1 parentChildInfo newName + assert parent1Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() + assert parentChildRelV1.getChildren(catalogName, databaseName, renameParent1) == + [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child11", "CLONE", "c11_uuid"), + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid") + ] as Set + // Test Child11 parentChildInfo + assert !child11Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + JSONAssert.assertEquals(child11ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child11).isEmpty() + + // Test Child12 parentChildInfo + assert !child12Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + JSONAssert.assertEquals(child12ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child12).isEmpty() + + //get the parent oldName should fail as it no longer exists + when: + api.getTable(catalogName, databaseName, parent1, true, true, false) + then: + e = thrown(Exception) + assert e.message.contains("Unable to locate for") + + /* + Step 10: Rename child11 to renameChild11 + */ + when: + api.renameTable(catalogName, databaseName, child11, renameChild11) + parent1Table = api.getTable(catalogName, databaseName, renameParent1, true, true, false) + child11Table = api.getTable(catalogName, databaseName, renameChild11, true, true, false) + child11ParentChildRelationInfo = child11Table.definitionMetadata.get("parentChildRelationInfo") + then: + // Test parent1Table parentChildInfo with newName + assert parent1Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() + assert parentChildRelV1.getChildren(catalogName, databaseName, renameParent1) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/rename_child11", "CLONE", "c11_uuid"), + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid") + ] as Set + // Test Child11 parentChildInfo with newName + assert !child11Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + assert child11Table.definitionMetadata.get("random_key").asText() == "random_value" + JSONAssert.assertEquals(child11ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, renameChild11).isEmpty() + + //get the child oldName should fail as it no longer exists + when: + api.getTable(catalogName, databaseName, child11, true, true, false) + then: + e = thrown(Exception) + assert e.message.contains("Unable to locate for") + + /* + Step 11: Drop parent renameParent1 + */ + when: + api.deleteTable(catalogName, databaseName, renameParent1) + + then: + e = thrown(Exception) + assert e.message.contains("because it still has") + + /* + Step 8: Drop renameChild11 should succeed + */ + when: + api.deleteTable(catalogName, databaseName, renameChild11) + parent1Table = api.getTable(catalogName, databaseName, renameParent1, true, true, false) + + then: + // Test parent1 Table + assert parent1Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() + assert parentChildRelV1.getChildren(catalogName, databaseName, renameParent1) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid") + ] as Set + + /* + Step 12: Create renameChild11 and should expect random_key should appear at it is reattached + but parent childInfo should not as it is always coming from the parent child relationship service + which currently does not have any record + */ + when: + child11TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, renameChild11, 'amajumdar', child11Uri) + api.createTable(catalogName, databaseName, renameChild11, child11TableDto) + child11Table = api.getTable(catalogName, databaseName, renameChild11, true, true, false) + then: + assert !child11Table.definitionMetadata.has("parentChildRelationInfo") + assert child11Table.definitionMetadata.get("random_key").asText() == "random_value" + assert parentChildRelV1.getChildren(catalogName, databaseName, child11).isEmpty() + + // Test parent1 Table still only have child12 + assert parent1Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() + assert parentChildRelV1.getChildren(catalogName, databaseName, renameParent1) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid") + ] as Set + + /* + Step 13: Drop child12 should succeed + */ + when: + api.deleteTable(catalogName, databaseName, child12) + parent1Table = api.getTable(catalogName, databaseName, renameParent1, true, true, false) + then: + assert !parent1Table.definitionMetadata.has("parentChildRelationInfo") + assert parentChildRelV1.getChildren(catalogName, databaseName, renameParent1).isEmpty() + + /* + Step 14: Drop renameParent1 should succeed as there is no more child under it + */ + when: + api.deleteTable(catalogName, databaseName, renameParent1) + parent2Table = api.getTable(catalogName, databaseName, parent2, true, true, false) + child21Table = api.getTable(catalogName, databaseName, child21, true, true, false) + child21ParentChildRelationInfo = child21Table.definitionMetadata.get("parentChildRelationInfo") + + then: + // Since all the operations above are on the first connected relationship, the second connected relationship + // should remain the same + // Test Parent 2 parentChildInfo + assert parent2Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() + assert parentChildRelV1.getChildren(catalogName, databaseName, parent2) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child21", "CLONE", "c21_uuid") + ] as Set + + // Test Child21 parentChildInfo + assert !child21Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + JSONAssert.assertEquals(child21ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent2","relationType":"CLONE","uuid":"p2_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child21).isEmpty() + + /* + Step 15: update parent2 with random parentChildRelationInfo to test immutability + */ + when: + def updateParent2Dto = parent2Table + initializeParentChildRelDefinitionMetadata(updateParent2Dto, "RANDOM", "RANDOM", "RANDOM") + api.updateTable(catalogName, databaseName, parent2, updateParent2Dto) + + parent2Table = api.getTable(catalogName, databaseName, parent2, true, true, false) + child21Table = api.getTable(catalogName, databaseName, child21, true, true, false) + child21ParentChildRelationInfo = child21Table.definitionMetadata.get("parentChildRelationInfo") + then: + assert parent2Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() + assert parentChildRelV1.getChildren(catalogName, databaseName, parent2) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child21", "CLONE", "c21_uuid") + ] as Set + + // Test Child21 parentChildInfo + assert !child21Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + JSONAssert.assertEquals(child21ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent2","relationType":"CLONE","uuid":"p2_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child21).isEmpty() + + /* + Step 16: update child21 with random parentChildRelationInfo to test immutability + */ + when: + def updateChild21Dto = child21Table + initializeParentChildRelDefinitionMetadata(updateParent2Dto, "RANDOM", "RANDOM", "RANDOM") + api.updateTable(catalogName, databaseName, child21, updateChild21Dto) + + parent2Table = api.getTable(catalogName, databaseName, parent2, true, true, false) + child21Table = api.getTable(catalogName, databaseName, child21, true, true, false) + child21ParentChildRelationInfo = child21Table.definitionMetadata.get("parentChildRelationInfo") + then: + // Test Parent 2 parentChildInfo + assert parent2Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() + assert parentChildRelV1.getChildren(catalogName, databaseName, parent2) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child21", "CLONE", "c21_uuid") + ] as Set + + // Test Child21 parentChildInfo + assert !child21Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + JSONAssert.assertEquals(child21ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent2","relationType":"CLONE","uuid":"p2_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child21).isEmpty() + } } diff --git a/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/ParentChildRelMetadataServiceSpec.groovy b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/ParentChildRelMetadataServiceSpec.groovy new file mode 100644 index 000000000..99f0d0284 --- /dev/null +++ b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/ParentChildRelMetadataServiceSpec.groovy @@ -0,0 +1,355 @@ +package com.netflix.metacat + +import com.netflix.metacat.common.QualifiedName +import com.netflix.metacat.common.server.converter.ConverterUtil +import com.netflix.metacat.common.server.converter.DefaultTypeConverter +import com.netflix.metacat.common.server.converter.DozerJsonTypeConverter +import com.netflix.metacat.common.server.converter.DozerTypeConverter +import com.netflix.metacat.common.server.converter.TypeConverterFactory +import com.netflix.metacat.common.server.model.ChildInfo +import com.netflix.metacat.common.server.model.ParentInfo +import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService +import com.netflix.metacat.metadata.mysql.MySqlParentChildRelMetaDataService +import org.springframework.jdbc.core.JdbcTemplate +import org.springframework.jdbc.datasource.DriverManagerDataSource +import spock.lang.Shared +import spock.lang.Specification + +class ParentChildRelMetadataServiceSpec extends Specification{ + @Shared + private ParentChildRelMetadataService service; + @Shared + private JdbcTemplate jdbcTemplate; + + @Shared + private ConverterUtil converterUtil; + + @Shared + private String catalog = "prodhive" + @Shared + private String database = "testpc" + + def setupSpec() { + String jdbcUrl = "jdbc:mysql://localhost:3306/metacat" + String username = "metacat_user" + String password = "metacat_user_password" + + DriverManagerDataSource dataSource = new DriverManagerDataSource() + dataSource.setDriverClassName("com.mysql.cj.jdbc.Driver") + dataSource.setUrl(jdbcUrl) + dataSource.setUsername(username) + dataSource.setPassword(password) + + jdbcTemplate = new JdbcTemplate(dataSource) + def typeFactory = new TypeConverterFactory(new DefaultTypeConverter()) + def converter = new ConverterUtil(new DozerTypeConverter(typeFactory), new DozerJsonTypeConverter(typeFactory)) + service = new MySqlParentChildRelMetaDataService(jdbcTemplate, converter) + } + + def cleanup() { + jdbcTemplate.update("DELETE FROM parent_child_relation") + } + + def "Test CreateThenDelete - OneChildOneParent"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def child = QualifiedName.ofTable(catalog, database, "c") + def parentUUID = "p_uuid"; + def childUUID = "c_uuid"; + def type = "clone"; + def child_parent_expected = [new ParentInfo(parent.toString(), type, parentUUID)] as Set + def parent_children_expected = [new ChildInfo(child.toString(), type, childUUID)] as Set + + when: + service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + + then: + // Test Parent + assert service.getParents(parent).isEmpty() + assert service.getChildren(parent) == parent_children_expected + + // Test Child + assert service.getParents(child) == child_parent_expected + assert service.getParents(child) == child_parent_expected + + when: + service.deleteParentChildRelation(parent, parentUUID, child, childUUID, type) + + then: + // Test Parent + assert service.getParents(parent).isEmpty() + assert service.getChildren(parent).isEmpty() + + // Test Child + assert service.getParents(child).isEmpty() + assert service.getParents(child).isEmpty() + + } + + def "Test Create - oneParentMultiChildren"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def parentUUID = "p_uuid"; + def child1 = QualifiedName.ofTable(catalog, database, "c1") + def child1UUID = "c1_uuid"; + def child2 = QualifiedName.ofTable(catalog, database, "c2") + def child2UUID = "c2_uuid"; + def type = "clone"; + def parent_children_expected = [ + new ChildInfo(child1.toString(), type, child1UUID), + new ChildInfo(child2.toString(), type, child2UUID), + ] as Set + def child_parent_expected = [new ParentInfo(parent.toString(), type, parentUUID)] as Set + + when: + service.createParentChildRelation(parent, parentUUID, child1, child1UUID, type) + service.createParentChildRelation(parent, parentUUID, child2, child2UUID, type) + + then: + // Test Parent + assert service.getParents(parent).isEmpty() + assert parent_children_expected == service.getChildren(parent) + + // Test Children + // Test Child 1 + assert child_parent_expected == service.getParents(child1) + assert service.getChildren(child1).isEmpty() + + assert child_parent_expected == service.getParents(child2) + assert service.getChildren(child2).isEmpty() + } + + def "Test Create - oneChildMultiParentException"() { + setup: + def parent1 = QualifiedName.ofTable(catalog, database, "p1") + def parent1UUID = "p1_uuid"; + def parent2 = QualifiedName.ofTable(catalog, database, "p2") + def parent2UUID = "p2_uuid"; + def child = QualifiedName.ofTable(catalog, database, "c") + def childUUID = "c_uuid" + def type = "clone" + service.createParentChildRelation(parent1, parent1UUID, child, childUUID, type) + + when: + service.createParentChildRelation(parent2, parent2UUID, child, childUUID, type) + + then: + def e = thrown(RuntimeException) + assert e.message.contains("Cannot have a child table having more than one parent") + + // Test Child + def child_parent_expected = [new ParentInfo(parent1.toString(), type, parent1UUID)] as Set + assert child_parent_expected == service.getParents(child) + assert service.getChildren(child).isEmpty() + } + + def "Test Create - oneChildAsParentOfAnotherException"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def parentUUID = "p_uuid" + def child = QualifiedName.ofTable(catalog, database, "c") + def childUUID = "c_uuid" + def grandChild = QualifiedName.ofTable(catalog, database, "gc") + def grandChildUUID = "gc_uuid" + def type = "clone" + service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + + when: + service.createParentChildRelation(child, childUUID, grandChild, grandChildUUID, type) + + then: + def e = thrown(RuntimeException) + assert e.message.contains("Cannot create a child table as parent") + + // Test Child + def child_parent_expected = [new ParentInfo(parent.toString(), type, parentUUID)] as Set + assert service.getParents(child) == child_parent_expected + assert service.getChildren(child).isEmpty() + } + + def "Test Create - oneParentAsChildOfAnother"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def parentUUID = "p_uuid" + def child = QualifiedName.ofTable(catalog, database, "c") + def childUUID = "c_uuid" + def grandChild = QualifiedName.ofTable(catalog, database, "gc") + def grandChildUUID = "gc_uuid" + def type = "clone" + service.createParentChildRelation(child, childUUID, grandChild, grandChildUUID, type) + + when: + service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + + then: + def e = thrown(RuntimeException) + assert e.message.contains("Cannot create a parent table on top of another parent") + } + + def "Test Rename Parent - One Child"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def parentUUID = "p_uuid" + def child = QualifiedName.ofTable(catalog, database, "c") + def childUUID = "c_uuid" + def type = "clone"; + def newParent = QualifiedName.ofTable(catalog, database, "np") + service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + + when: + service.rename(parent, newParent) + + then: + // Test Old Parent Name + assert service.getParents(parent).isEmpty() + assert service.getChildren(parent).isEmpty() + + // Test New Parent Name + assert service.getParents(newParent).isEmpty() + def newParent_children_expected = [new ChildInfo(child.toString(), type, childUUID)] as Set + assert service.getChildren(newParent) == newParent_children_expected + + // Test Child + def child_parent_expected = [new ParentInfo(newParent.toString(), type, parentUUID)] as Set + assert child_parent_expected == service.getParents(child) + assert service.getChildren(child).isEmpty() + + // rename back + when: + service.rename(newParent, parent) + child_parent_expected = [new ParentInfo(parent.toString(), type, parentUUID)] as Set + + then: + // Test new Parent Name + assert service.getParents(newParent).isEmpty() + assert service.getChildren(newParent).isEmpty() + + // Test old Parent Name + assert service.getParents(parent).isEmpty() + assert service.getChildren(parent) == newParent_children_expected + + // Test Child + assert child_parent_expected == service.getParents(child) + assert service.getChildren(child).isEmpty() + } + + def "Test Rename Parent - Multi Child"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def parentUUID = "p_uuid" + def child1 = QualifiedName.ofTable(catalog, database, "c1") + def child1UUID = "c1_uuid" + def child2 = QualifiedName.ofTable(catalog, database, "c2") + def child2UUID = "c2_uuid" + def type = "clone"; + service.createParentChildRelation(parent, parentUUID, child1, child1UUID, type) + service.createParentChildRelation(parent, parentUUID, child2, child2UUID, type) + def newParent = QualifiedName.ofTable(catalog, database, "np") + def child_parent_expected = [new ParentInfo(newParent.toString(), type, parentUUID)] as Set + + when: + service.rename(parent, newParent) + + then: + // Test Child1 + assert service.getParents(child1) == child_parent_expected + //Test Child2 + assert service.getParents(child2) == child_parent_expected + } + + def "Test Rename Child"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def parentUUID = "p_uuid" + def child = QualifiedName.ofTable(catalog, database, "c") + def childUUID = "c_uuid" + def type = "clone" + service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + def newChild = QualifiedName.ofTable(catalog, database, "nc") + + when: + service.rename(child, newChild) + then: + // Test Parent + assert service.getParents(parent).isEmpty() + def parent_children_expected = [new ChildInfo(newChild.toString(), type, childUUID)] as Set + assert parent_children_expected == service.getChildren(parent) + + // Test Child + assert service.getParents(child).isEmpty() + assert service.getChildren(child).isEmpty() + + // Test New Child + def child_parent_expected = [new ParentInfo(parent.toString(), type, parentUUID)] as Set + assert child_parent_expected == service.getParents(newChild) + assert service.getChildren(child).isEmpty() + + // rename back + when: + service.rename(newChild, child) + parent_children_expected = [new ChildInfo(child.toString(), type, childUUID)] as Set + + then: + // Test Parent + assert service.getParents(parent).isEmpty() + assert parent_children_expected == service.getChildren(parent) + + // Test New Child + assert service.getParents(newChild).isEmpty() + assert service.getChildren(newChild).isEmpty() + + // Test Child + assert child_parent_expected == service.getParents(child) + assert service.getChildren(child).isEmpty() + } + + + def "Test Drop Child"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def parentUUID = "p_uuid" + def child = QualifiedName.ofTable(catalog, database, "c") + def childUUID = "c_uuid" + def type = "clone"; + service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + when: + service.drop(child) + + then: + // Test Parent + assert service.getParents(parent).isEmpty() + assert service.getChildren(parent).isEmpty() + + // Test Child + assert service.getParents(child).isEmpty() + assert service.getChildren(child).isEmpty() + } + + def "Test Rename and Drop Child"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def parentUUID = "p_uuid" + def child = QualifiedName.ofTable(catalog, database, "c") + def newChild = QualifiedName.ofTable(catalog, database, "nc") + def childUUID = "c_uuid" + def type = "clone"; + service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + + when: + service.rename(child, newChild) + service.drop(newChild) + + then: + // Test Parent + assert service.getParents(parent).isEmpty() + assert service.getChildren(parent).isEmpty() + + // Test Child + assert service.getParents(child).isEmpty() + assert service.getChildren(child).isEmpty() + + // Test newChild + assert service.getParents(newChild).isEmpty() + assert service.getChildren(newChild).isEmpty() + } +} diff --git a/metacat-main/src/main/java/com/netflix/metacat/main/api/v1/ParentChildRelController.java b/metacat-main/src/main/java/com/netflix/metacat/main/api/v1/ParentChildRelController.java new file mode 100644 index 000000000..6a43a70d2 --- /dev/null +++ b/metacat-main/src/main/java/com/netflix/metacat/main/api/v1/ParentChildRelController.java @@ -0,0 +1,75 @@ +package com.netflix.metacat.main.api.v1; + +import com.netflix.metacat.common.QualifiedName; +import com.netflix.metacat.common.dto.notifications.ChildInfoDto; +import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService; +import com.netflix.metacat.main.api.RequestWrapper; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import lombok.RequiredArgsConstructor; +import org.springframework.context.annotation.DependsOn; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.PathVariable; + +import java.util.Set; + +/** + * Parent Child Relation V1 API implementation. + * + * @author Yingjian + */ + +@RestController +@RequestMapping( + path = "/mds/v1/parentChildRel", + produces = MediaType.APPLICATION_JSON_VALUE +) +@Api( + value = "ParentChildRelV1", + description = "Federated user metadata operations", + produces = MediaType.APPLICATION_JSON_VALUE, + consumes = MediaType.APPLICATION_JSON_VALUE +) +@DependsOn("metacatCoreInitService") +@RequiredArgsConstructor +public class ParentChildRelController { + private final RequestWrapper requestWrapper; + private final ParentChildRelMetadataService parentChildRelMetadataService; + + /** + * Return the list of children for a given table. + * @param catalogName catalogName + * @param databaseName databaseName + * @param tableName tableName + * @return list of childInfoDto + */ + @RequestMapping(method = RequestMethod.GET, + path = "/children/catalog/{catalog-name}/database/{database-name}/table/{table-name}") + @ResponseStatus(HttpStatus.OK) + @ApiOperation( + position = 0, + value = "Returns the children", + notes = "Returns the children" + ) + public Set getChildren( + @ApiParam(value = "The name of the catalog", required = true) + @PathVariable("catalog-name") final String catalogName, + @ApiParam(value = "The name of the database", required = true) + @PathVariable("database-name") final String databaseName, + @ApiParam(value = "The name of the table", required = true) + @PathVariable("table-name") final String tableName + ) { + return this.requestWrapper.processRequest( + "ParentChildRelV1Resource.getChildren", + () -> this.parentChildRelMetadataService.getChildrenDto( + QualifiedName.ofTable(catalogName, databaseName, tableName) + ) + ); + } +} diff --git a/metacat-main/src/main/java/com/netflix/metacat/main/configs/ServicesConfig.java b/metacat-main/src/main/java/com/netflix/metacat/main/configs/ServicesConfig.java index 02dd8ac9a..b15f751db 100644 --- a/metacat-main/src/main/java/com/netflix/metacat/main/configs/ServicesConfig.java +++ b/metacat-main/src/main/java/com/netflix/metacat/main/configs/ServicesConfig.java @@ -35,6 +35,7 @@ import com.netflix.metacat.common.server.usermetadata.LookupService; import com.netflix.metacat.common.server.usermetadata.TagService; import com.netflix.metacat.common.server.usermetadata.UserMetadataService; +import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService; import com.netflix.metacat.common.server.util.ThreadServiceManager; import com.netflix.metacat.main.manager.CatalogManager; import com.netflix.metacat.main.manager.ConnectorManager; @@ -228,6 +229,7 @@ public DatabaseService databaseService( * @param converterUtil converter utilities * @param authorizationService authorization Service * @param ownerValidationService owner validation service + * @param parentChildRelMetadataService parentChildRelMetadataService * * @return The table service bean */ @@ -244,7 +246,8 @@ public TableService tableService( final Config config, final ConverterUtil converterUtil, final AuthorizationService authorizationService, - final OwnerValidationService ownerValidationService) { + final OwnerValidationService ownerValidationService, + final ParentChildRelMetadataService parentChildRelMetadataService) { return new TableServiceImpl( connectorManager, connectorTableServiceProxy, @@ -257,7 +260,8 @@ public TableService tableService( config, converterUtil, authorizationService, - ownerValidationService + ownerValidationService, + parentChildRelMetadataService ); } diff --git a/metacat-main/src/main/java/com/netflix/metacat/main/services/impl/TableServiceImpl.java b/metacat-main/src/main/java/com/netflix/metacat/main/services/impl/TableServiceImpl.java index 3ea1c41a2..32a6bc8b8 100644 --- a/metacat-main/src/main/java/com/netflix/metacat/main/services/impl/TableServiceImpl.java +++ b/metacat-main/src/main/java/com/netflix/metacat/main/services/impl/TableServiceImpl.java @@ -13,6 +13,9 @@ package com.netflix.metacat.main.services.impl; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -42,14 +45,18 @@ import com.netflix.metacat.common.server.events.MetacatUpdateIcebergTablePostEvent; import com.netflix.metacat.common.server.events.MetacatUpdateTablePostEvent; import com.netflix.metacat.common.server.events.MetacatUpdateTablePreEvent; +import com.netflix.metacat.common.server.model.ChildInfo; +import com.netflix.metacat.common.server.model.ParentInfo; import com.netflix.metacat.common.server.monitoring.Metrics; import com.netflix.metacat.common.server.properties.Config; import com.netflix.metacat.common.server.spi.MetacatCatalogConfig; import com.netflix.metacat.common.server.usermetadata.AuthorizationService; import com.netflix.metacat.common.server.usermetadata.GetMetadataInterceptorParameters; +import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataConstants; import com.netflix.metacat.common.server.usermetadata.MetacatOperation; import com.netflix.metacat.common.server.usermetadata.TagService; import com.netflix.metacat.common.server.usermetadata.UserMetadataService; +import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService; import com.netflix.metacat.common.server.util.MetacatContextManager; import com.netflix.metacat.common.server.util.MetacatUtils; import com.netflix.metacat.main.manager.ConnectorManager; @@ -62,7 +69,6 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; - import javax.annotation.Nullable; import java.util.List; import java.util.Map; @@ -71,6 +77,8 @@ import java.util.Set; import java.util.concurrent.TimeUnit; + + /** * Table service implementation. */ @@ -89,6 +97,7 @@ public class TableServiceImpl implements TableService { private final ConverterUtil converterUtil; private final AuthorizationService authorizationService; private final OwnerValidationService ownerValidationService; + private final ParentChildRelMetadataService parentChildRelMetadataService; /** * {@inheritDoc} @@ -106,7 +115,13 @@ public TableDto create(final QualifiedName name, final TableDto tableDto) { log.info("Creating table {}", name); eventBus.post(new MetacatCreateTablePreEvent(name, metacatRequestContext, this, tableDto)); - connectorTableServiceProxy.create(name, converterUtil.fromTableDto(tableDto)); + final Optional unSaveOpOpt = saveParentChildRelationship(name, tableDto); + try { + connectorTableServiceProxy.create(name, converterUtil.fromTableDto(tableDto)); + } catch (Exception e) { + unSaveOpOpt.ifPresent(Runnable::run); + throw e; + } if (tableDto.getDataMetadata() != null || tableDto.getDefinitionMetadata() != null) { log.info("Saving user metadata for table {}", name); @@ -139,6 +154,91 @@ public TableDto create(final QualifiedName name, final TableDto tableDto) { return dto; } + private ObjectNode createParentChildObjectNode(@Nullable final Set parentInfos, + @Nullable final Set childInfos) { + final ObjectMapper objectMapper = new ObjectMapper(); + final ObjectNode rootNode = objectMapper.createObjectNode(); + + // For childTable, we will always populate the parent infos + if (parentInfos != null && !parentInfos.isEmpty()) { + final ArrayNode parentArrayNode = objectMapper.createArrayNode(); + for (ParentInfo parentInfo : parentInfos) { + final ObjectNode parentNode = objectMapper.createObjectNode(); + parentNode.put("name", parentInfo.getName()); + parentNode.put("relationType", parentInfo.getRelationType()); + parentNode.put("uuid", parentInfo.getUuid()); + parentArrayNode.add(parentNode); + } + rootNode.set(ParentChildRelMetadataConstants.PARENT_INFOS, parentArrayNode); + } + + // For parent table, if it has a child, we will put a field to indicate that the table is a parent table. + if (childInfos != null && !childInfos.isEmpty()) { + rootNode.put(ParentChildRelMetadataConstants.IS_PARENT, true); + } + return rootNode; + } + + // Return the unSaveOperation if applicable + private Optional saveParentChildRelationship(final QualifiedName child, final TableDto tableDto) { + if (tableDto.getDefinitionMetadata() != null) { + final ObjectNode definitionMetadata = tableDto.getDefinitionMetadata(); + if (definitionMetadata.has(ParentChildRelMetadataConstants.PARENT_CHILD_RELINFO)) { + final JsonNode parentChildRelInfo = + definitionMetadata.get(ParentChildRelMetadataConstants.PARENT_CHILD_RELINFO); + + String parentName; + if (!parentChildRelInfo.has(ParentChildRelMetadataConstants.PARENT_NAME)) { + throw new RuntimeException("parent name is not specified"); + } + parentName = parentChildRelInfo.path(ParentChildRelMetadataConstants.PARENT_NAME) + .asText(); + final QualifiedName parent = QualifiedName.fromString(parentName); + validate(parent); + + // fetch parent and child uuid + String parentUUID; + String childUUID; + if (!parentChildRelInfo.has(ParentChildRelMetadataConstants.PARENT_UUID)) { + throw new RuntimeException("root_table_uuid is not specified for parent table=" + + parentName); + } + + if (!parentChildRelInfo.has(ParentChildRelMetadataConstants.CHILD_UUID)) { + throw new RuntimeException("child_table_uuid is not specified for child table=" + child); + } + parentUUID = parentChildRelInfo.path(ParentChildRelMetadataConstants.PARENT_UUID).asText(); + childUUID = parentChildRelInfo.path(ParentChildRelMetadataConstants.CHILD_UUID).asText(); + + // fetch relationshipType + String relationType; + if (parentChildRelInfo.has(ParentChildRelMetadataConstants.RELATION_TYPE)) { + relationType = parentChildRelInfo.path(ParentChildRelMetadataConstants.RELATION_TYPE).asText(); + } else { + relationType = "CLONE"; + } + + // Create parent child relationship + parentChildRelMetadataService.createParentChildRelation(parent, parentUUID, + child, childUUID, relationType); + + // Return a Runnable for deleting the relationship + return Optional.of(() -> { + try { + parentChildRelMetadataService.deleteParentChildRelation(parent, + parentUUID, child, childUUID, relationType); + } catch (Exception e) { + log.error("parentChildRelMetadataService: Failed to delete parent child relationship " + + "after failing to create the table={} with the following parameters: " + + "parent={}, parentUUID={}, child={}, childUUID={}, relationType={}", + child, parent, parentUUID, child, childUUID, relationType, e); + } + }); + } + } + return Optional.empty(); + } + private void setDefaultAttributes(final TableDto tableDto) { setDefaultSerdeIfNull(tableDto); setDefaultDefinitionMetadataIfNull(tableDto); @@ -267,10 +367,20 @@ public TableDto deleteAndReturn(final QualifiedName name, final boolean isMView) } } - // Try to delete the table even if get above fails + final Set childInfos = parentChildRelMetadataService.getChildren(name); + if (childInfos != null && !childInfos.isEmpty()) { + final StringBuilder errorSb = new StringBuilder(); + errorSb.append("Failed to drop ").append(name) + .append(" because it still has ") + .append(childInfos.size()) + .append(" child table(s). One example is:"); + + errorSb.append("\n").append(childInfos.iterator().next().getName()); + + throw new RuntimeException(errorSb.toString()); + } try { connectorTableServiceProxy.delete(name); - // If this is a common view, the storage_table if present // should also be deleted. if (MetacatUtils.isCommonView(tableDto.getMetadata()) @@ -283,11 +393,16 @@ public TableDto deleteAndReturn(final QualifiedName name, final boolean isMView) deleteCommonViewStorageTable(name, qualifiedStorageTableName); } } - } catch (NotFoundException ignored) { log.debug("NotFoundException ignored for table {}", name); } + try { + parentChildRelMetadataService.drop(name); + } catch (Exception e) { + log.error("parentChildRelMetadataService: Failed to drop relation for table={}", name, e); + } + if (canDeleteMetadata(name)) { // Delete the metadata. Type doesn't matter since we discard the result log.info("Deleting user metadata for table {}", name); @@ -373,11 +488,26 @@ public Optional get(final QualifiedName name, final GetTableServicePar } if (getTableServiceParameters.isIncludeDefinitionMetadata()) { - final Optional definitionMetadata = + Optional definitionMetadata = (getTableServiceParameters.isDisableOnReadMetadataIntercetor()) ? userMetadataService.getDefinitionMetadata(name) : userMetadataService.getDefinitionMetadataWithInterceptor(name, GetMetadataInterceptorParameters.builder().hasMetadata(tableInternal).build()); + // Always get the source of truth for parent child relation from the parentChildRelMetadataService + if (definitionMetadata.isPresent() + && definitionMetadata.get().has(ParentChildRelMetadataConstants.PARENT_CHILD_RELINFO)) { + definitionMetadata.get().remove(ParentChildRelMetadataConstants.PARENT_CHILD_RELINFO); + } + final Set parentInfo = parentChildRelMetadataService.getParents(name); + final Set childInfos = parentChildRelMetadataService.getChildren(name); + final ObjectNode parentChildRelObjectNode = createParentChildObjectNode(parentInfo, childInfos); + if (!parentChildRelObjectNode.isEmpty()) { + if (!definitionMetadata.isPresent()) { + definitionMetadata = Optional.of(new ObjectMapper().createObjectNode()); + } + definitionMetadata.get().set(ParentChildRelMetadataConstants.PARENT_CHILD_RELINFO, + parentChildRelObjectNode); + } definitionMetadata.ifPresent(table::setDefinitionMetadata); } @@ -437,7 +567,24 @@ public void rename( if (oldTable != null) { //Ignore if the operation is not supported, so that we can at least go ahead and save the user metadata eventBus.post(new MetacatRenameTablePreEvent(oldName, metacatRequestContext, this, newName)); - connectorTableServiceProxy.rename(oldName, newName, isMView); + + // Before rename, first rename its parent child relation + parentChildRelMetadataService.rename(oldName, newName); + + try { + connectorTableServiceProxy.rename(oldName, newName, isMView); + } catch (Exception e) { + try { + // if rename operation fail, rename back the parent child relation + parentChildRelMetadataService.rename(newName, oldName); + } catch (Exception renameException) { + log.error("parentChildRelMetadataService: Failed to rename parent child relation " + + "after table fail to rename from {} to {} " + + "with the following parameters oldName={} to newName={}", + oldName, newName, oldName, newName, renameException); + } + throw e; + } userMetadataService.renameDefinitionMetadataKey(oldName, newName); tagService.renameTableTags(oldName, newName.getTableName()); diff --git a/metacat-main/src/test/groovy/com/netflix/metacat/main/services/impl/TableServiceImplSpec.groovy b/metacat-main/src/test/groovy/com/netflix/metacat/main/services/impl/TableServiceImplSpec.groovy index 286c78696..be3366222 100644 --- a/metacat-main/src/test/groovy/com/netflix/metacat/main/services/impl/TableServiceImplSpec.groovy +++ b/metacat-main/src/test/groovy/com/netflix/metacat/main/services/impl/TableServiceImplSpec.groovy @@ -35,9 +35,12 @@ import com.netflix.metacat.common.server.converter.ConverterUtil import com.netflix.metacat.common.server.events.MetacatEventBus import com.netflix.metacat.common.server.events.MetacatUpdateTablePostEvent import com.netflix.metacat.common.server.events.MetacatUpdateTablePreEvent +import com.netflix.metacat.common.server.model.ChildInfo +import com.netflix.metacat.common.server.model.ParentInfo import com.netflix.metacat.common.server.properties.Config import com.netflix.metacat.common.server.spi.MetacatCatalogConfig import com.netflix.metacat.common.server.usermetadata.DefaultAuthorizationService +import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataConstants import com.netflix.metacat.common.server.usermetadata.TagService import com.netflix.metacat.common.server.usermetadata.UserMetadataService import com.netflix.metacat.common.server.util.MetacatContextManager @@ -50,8 +53,7 @@ import com.netflix.spectator.api.DefaultRegistry import com.netflix.spectator.api.NoopRegistry import spock.lang.Specification import spock.lang.Unroll - -import javax.annotation.meta.When +import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService; /** * Tests for the TableServiceImpl. @@ -80,6 +82,7 @@ class TableServiceImplSpec extends Specification { def connectorTableServiceProxy def authorizationService def ownerValidationService + def parentChildRelSvc def service def setup() { @@ -93,13 +96,14 @@ class TableServiceImplSpec extends Specification { usermetadataService.getDefinitionMetadata(_) >> Optional.empty() usermetadataService.getDataMetadata(_) >> Optional.empty() usermetadataService.getDefinitionMetadataWithInterceptor(_,_) >> Optional.empty() - connectorTableServiceProxy = new ConnectorTableServiceProxy(connectorManager, converterUtil) + connectorTableServiceProxy = Spy(new ConnectorTableServiceProxy(connectorManager, converterUtil)) authorizationService = new DefaultAuthorizationService(config) ownerValidationService = Mock(OwnerValidationService) + parentChildRelSvc = Mock(ParentChildRelMetadataService) service = new TableServiceImpl(connectorManager, connectorTableServiceProxy, databaseService, tagService, usermetadataService, new MetacatJsonLocator(), - eventBus, registry, config, converterUtil, authorizationService, ownerValidationService) + eventBus, registry, config, converterUtil, authorizationService, ownerValidationService, parentChildRelSvc) } def testTableGet() { @@ -270,6 +274,76 @@ class TableServiceImplSpec extends Specification { 0 * ownerValidationService.enforceOwnerValidation(_, _, _) } + def "Test Create - Clone Table Fail to create table"() { + given: + def childTableName = QualifiedName.ofTable("clone", "clone", "c") + def parentTableName = QualifiedName.ofTable("clone", "clone", "p") + def mapper = new ObjectMapper() + + def innerNode = mapper.createObjectNode() + innerNode.put("root_table_name", "clone/clone/p") + innerNode.put("root_table_uuid", "p_uuid") + innerNode.put("child_table_uuid", "child_uuid") + + def outerNode = mapper.createObjectNode() + outerNode.set(ParentChildRelMetadataConstants.PARENT_CHILD_RELINFO, innerNode) + def createTableDto = new TableDto( + name: childTableName, + definitionMetadata: outerNode, + serde: new StorageDto(uri: 's3:/clone/clone/c') + ) + when: + service.create(childTableName, createTableDto) + then: + 1 * ownerValidationService.extractPotentialOwners(_) >> ["cloneClient"] + 1 * ownerValidationService.isUserValid(_) >> true + 1 * ownerValidationService.extractPotentialOwnerGroups(_) >> ["cloneClientGroup"] + 1 * ownerValidationService.isGroupValid(_) >> true + + 1 * parentChildRelSvc.createParentChildRelation(parentTableName, "p_uuid", childTableName, "child_uuid", "CLONE") + 1 * connectorTableServiceProxy.create(_, _) >> {throw new RuntimeException("Fail to create")} + 1 * parentChildRelSvc.deleteParentChildRelation(parentTableName, "p_uuid", childTableName, "child_uuid", "CLONE") + thrown(RuntimeException) + } + + def "Test Rename - Clone Table Fail to update parent child relation"() { + given: + def oldName = QualifiedName.ofTable("clone", "clone", "oldChild") + def newName = QualifiedName.ofTable("clone", "clone", "newChild") + when: + service.rename(oldName, newName, false) + + then: + 1 * config.getNoTableRenameOnTags() >> [] + 1 * parentChildRelSvc.rename(oldName, newName) + 1 * connectorTableServiceProxy.rename(oldName, newName, _) >> {throw new RuntimeException("Fail to rename")} + 1 * parentChildRelSvc.rename(newName, oldName) + thrown(RuntimeException) + } + + def "Test Drop - Clone Table Fail to drop parent child relation"() { + given: + def name = QualifiedName.ofTable("clone", "clone", "child") + + when: + service.delete(name) + then: + 1 * parentChildRelSvc.getParents(name) >> {[new ParentInfo("parent", "clone", "parent_uuid")] as Set} + 2 * parentChildRelSvc.getChildren(name) >> {[new ChildInfo("child", "clone", "child_uuid")] as Set} + 1 * config.getNoTableDeleteOnTags() >> [] + thrown(RuntimeException) + + when: + service.delete(name) + then: + 1 * parentChildRelSvc.getParents(name) + 2 * parentChildRelSvc.getChildren(name) + 1 * config.getNoTableDeleteOnTags() >> [] + 1 * connectorTableServiceProxy.delete(_) >> {throw new RuntimeException("Fail to drop")} + 0 * parentChildRelSvc.drop(_, _) + thrown(RuntimeException) + } + def "Will not throw on Successful Table Update with Failed Get"() { given: def updatedTableDto = new TableDto(name: name, serde: new StorageDto(uri: 's3:/a/b/c')) @@ -311,7 +385,7 @@ class TableServiceImplSpec extends Specification { connectorManager, connectorTableServiceProxy, databaseService, tagService, usermetadataService, new MetacatJsonLocator(), eventBus, new DefaultRegistry(), config, converterUtil, authorizationService, - new DefaultOwnerValidationService(new NoopRegistry())) + new DefaultOwnerValidationService(new NoopRegistry()), parentChildRelSvc) def initialDefinitionMetadataJson = toObjectNode(initialDefinitionMetadata) tableDto = new TableDto( diff --git a/metacat-metadata-mysql/build.gradle b/metacat-metadata-mysql/build.gradle index db158f794..c4291fa8e 100644 --- a/metacat-metadata-mysql/build.gradle +++ b/metacat-metadata-mysql/build.gradle @@ -32,6 +32,7 @@ dependencies { api("com.google.guava:guava") api("org.apache.tomcat:tomcat-jdbc") api("org.slf4j:slf4j-api") + api('org.springframework.retry:spring-retry') /******************************* * Provided Dependencies *******************************/ diff --git a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlParentChildRelMetaDataService.java b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlParentChildRelMetaDataService.java new file mode 100644 index 000000000..89f0ef6d6 --- /dev/null +++ b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlParentChildRelMetaDataService.java @@ -0,0 +1,264 @@ +package com.netflix.metacat.metadata.mysql; + +import com.netflix.metacat.common.QualifiedName; +import com.netflix.metacat.common.dto.notifications.ChildInfoDto; +import com.netflix.metacat.common.server.converter.ConverterUtil; +import com.netflix.metacat.common.server.model.ChildInfo; +import com.netflix.metacat.common.server.model.ParentInfo; +import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService; +import com.netflix.metacat.common.server.usermetadata.ParentChildRelServiceException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.PreparedStatement; +import java.util.Set; +import java.util.List; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.stream.Collectors; + +/** + * Parent Child Relationship Metadata Service. + * This stores the parent child relationship of two entities as first class citizen in Metacat. + */ +@Slf4j +@SuppressFBWarnings +@Transactional("metadataTxManager") +public class MySqlParentChildRelMetaDataService implements ParentChildRelMetadataService { + static final String SQL_CREATE_PARENT_CHILD_RELATIONS = + "INSERT INTO parent_child_relation (parent, parent_uuid, child, child_uuid, relation_type) " + + "VALUES (?, ?, ?, ?, ?)"; + + static final String SQL_DELETE_PARENT_CHILD_RELATIONS = + "DELETE FROM parent_child_relation " + + "WHERE parent = ? AND parent_uuid = ? AND child = ? AND child_uuid = ? AND relation_type = ?"; + + static final String SQL_RENAME_PARENT_ENTITY = "UPDATE parent_child_relation " + + "SET parent = ? WHERE parent = ?"; + static final String SQL_RENAME_CHILD_ENTITY = "UPDATE parent_child_relation " + + "SET child = ? WHERE child = ?"; + + static final String SQL_DROP_CHILD = "DELETE FROM parent_child_relation " + + "WHERE child = ? "; + static final String SQL_DROP_PARENT = "DELETE FROM parent_child_relation " + + "WHERE parent = ? "; + + static final String SQL_GET_PARENTS = "SELECT parent, parent_uuid, relation_type " + + "FROM parent_child_relation WHERE child = ?"; + + static final String SQL_GET_CHILDREN = "SELECT child, child_uuid, relation_type " + + "FROM parent_child_relation WHERE parent = ?"; + + private final JdbcTemplate jdbcTemplate; + private final ConverterUtil converterUtil; + + /** + * Constructor. + * + * @param jdbcTemplate jdbc template + * @param converterUtil converterUtil + */ + @Autowired + public MySqlParentChildRelMetaDataService(final JdbcTemplate jdbcTemplate, final ConverterUtil converterUtil) { + this.jdbcTemplate = jdbcTemplate; + this.converterUtil = converterUtil; + } + + @Override + public void createParentChildRelation(final QualifiedName parentName, + final String parentUUID, + final QualifiedName childName, + final String childUUID, + final String type) { + // Validation to prevent having a child have two parents + final Set childParents = getParents(childName); + if (!childParents.isEmpty()) { + throw new ParentChildRelServiceException("Cannot have a child table having more than one parent " + + "- Child Table: " + childName + + " already have a parent Table=" + childParents.stream().findFirst().get()); + } + + // Validation to prevent creating a child table as a parent of another child table + final Set parentParents = getParents(parentName); + if (!parentParents.isEmpty()) { + throw new ParentChildRelServiceException("Cannot create a child table as parent " + + "- parent table: " + parentName + + " already have a parent table = " + parentParents.stream().findFirst().get()); + } + + // Validation to prevent creating a parent on top of a table that have children + final Set childChildren = getChildren(childName); + if (!childChildren.isEmpty()) { + throw new ParentChildRelServiceException("Cannot create a parent table on top of another parent " + + "- child table: " + childName + " already have child"); + } + + try { + jdbcTemplate.update(connection -> { + final PreparedStatement ps = connection.prepareStatement(SQL_CREATE_PARENT_CHILD_RELATIONS); + ps.setString(1, parentName.toString()); + ps.setString(2, parentUUID); + ps.setString(3, childName.toString()); + ps.setString(4, childUUID); + ps.setString(5, type); + return ps; + }); + } catch (Exception e) { + log.error("Fail to create parent child relationship with parent={}, parentUUID={}, " + + "child={}, childUUID={}, relationType = {}", + parentName, parentUUID, childName, childUUID, type + ); + throw new ParentChildRelServiceException("Fail to create parent child relationship " + + "for child table:" + childName + " under parent table:" + parentName, e); + } + log.info("Successfully create parent child relationship with parent={}, parentUUID={}, " + + "child={}, childUUID={}, relationType = {}", + parentName, parentUUID, childName, childUUID, type + ); + } + + @Override + public void deleteParentChildRelation(final QualifiedName parentName, + final String parentUUID, + final QualifiedName childName, + final String childUUID, + final String type) { + log.info("Deleting parent child relationship with parent={}, parentUUID={}, " + + "child={}, childUUID={}, relationType = {}", + parentName, parentUUID, childName, childUUID, type + ); + + try { + jdbcTemplate.update(connection -> { + final PreparedStatement ps = connection.prepareStatement(SQL_DELETE_PARENT_CHILD_RELATIONS); + ps.setString(1, parentName.toString()); + ps.setString(2, parentUUID); + ps.setString(3, childName.toString()); + ps.setString(4, childUUID); + ps.setString(5, type); + return ps; + }); + } catch (Exception e) { + log.error("Fail to delete parent child relationship with parent={}, parentUUID={}, " + + "child={}, childUUID={}, relationType = {}", + parentName, parentUUID, childName, childUUID, type + ); + throw new ParentChildRelServiceException("Fail to delete parent child relationship" + + " for child table:" + childName + " under parent table:" + parentName, e); + } + log.info("Successfully delete parent child relationship with parent={}, parentUUID={}, " + + "child={}, childUUID={}, relationType = {}", + parentName, parentUUID, childName, childUUID, type + ); + } + + @Override + public void rename(final QualifiedName oldName, final QualifiedName newName) { + renameParent(oldName, newName); + renameChild(oldName, newName); + log.info("Successfully rename parent child relationship for oldName={}, newName={}", + oldName, newName + ); + } + + private void renameParent(final QualifiedName oldName, final QualifiedName newName) { + try { + jdbcTemplate.update(connection -> { + final PreparedStatement ps = connection.prepareStatement(SQL_RENAME_PARENT_ENTITY); + ps.setString(1, newName.toString()); + ps.setString(2, oldName.toString()); + + return ps; + }); + } catch (Exception e) { + throw new ParentChildRelServiceException("Fail to rename parent from oldName:" + oldName + " to " + newName + + " in MySqlParentChildRelMetadataService", e); + } + } + + private void renameChild(final QualifiedName oldName, final QualifiedName newName) { + try { + jdbcTemplate.update(connection -> { + final PreparedStatement ps = connection.prepareStatement(SQL_RENAME_CHILD_ENTITY); + ps.setString(1, newName.toString()); + ps.setString(2, oldName.toString()); + return ps; + }); + } catch (Exception e) { + throw new ParentChildRelServiceException("Fail to rename child from oldName:" + oldName + " to " + newName + + " in MySqlParentChildRelMetadataService", e); + } + } + + @Override + public void drop(final QualifiedName name) { + dropParent(name); + dropChild(name); + log.info("Successfully drop parent child relationship for name={}", name); + } + + private void dropParent(final QualifiedName name) { + try { + jdbcTemplate.update(connection -> { + final PreparedStatement ps = connection.prepareStatement(SQL_DROP_PARENT); + ps.setString(1, name.toString()); + return ps; + }); + } catch (Exception e) { + throw new ParentChildRelServiceException("Fail to drop parent:" + name + + " in MySqlParentChildRelMetadataService", e); + } + } + + private void dropChild(final QualifiedName name) { + try { + jdbcTemplate.update(connection -> { + final PreparedStatement ps = connection.prepareStatement(SQL_DROP_CHILD); + ps.setString(1, name.toString()); + return ps; + }); + } catch (Exception e) { + throw new ParentChildRelServiceException("Fail to drop child:" + name + + " in MySqlParentChildRelMetadataService", e); + } + } + + @Override + public Set getParents(final QualifiedName name) { + final List params = new ArrayList<>(); + params.add(name.toString()); + final List parents = jdbcTemplate.query( + SQL_GET_PARENTS, params.toArray(), (rs, rowNum) -> { + final ParentInfo parentInfo = new ParentInfo(); + parentInfo.setName(rs.getString("parent")); + parentInfo.setRelationType(rs.getString("relation_type")); + parentInfo.setUuid(rs.getString("parent_uuid")); + return parentInfo; + }); + return new HashSet<>(parents); + } + + @Override + public Set getChildren(final QualifiedName name) { + final List params = new ArrayList<>(); + params.add(name.toString()); + final List children = jdbcTemplate.query( + SQL_GET_CHILDREN, params.toArray(), (rs, rowNum) -> { + final ChildInfo childInfo = new ChildInfo(); + childInfo.setName(rs.getString("child")); + childInfo.setRelationType(rs.getString("relation_type")); + childInfo.setUuid(rs.getString("child_uuid")); + return childInfo; + }); + return new HashSet<>(children); + } + + @Override + public Set getChildrenDto(final QualifiedName name) { + return getChildren(name).stream() + .map(converterUtil::toChildInfoDto).collect(Collectors.toSet()); + } +} diff --git a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java index e5df7761d..a195d403c 100644 --- a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java +++ b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java @@ -14,13 +14,15 @@ package com.netflix.metacat.metadata.mysql; import com.netflix.metacat.common.json.MetacatJson; +import com.netflix.metacat.common.server.converter.ConverterUtil; import com.netflix.metacat.common.server.properties.Config; import com.netflix.metacat.common.server.properties.MetacatProperties; -import com.netflix.metacat.common.server.usermetadata.MetadataInterceptor; +import com.netflix.metacat.common.server.usermetadata.UserMetadataService; import com.netflix.metacat.common.server.usermetadata.LookupService; -import com.netflix.metacat.common.server.usermetadata.MetadataInterceptorImpl; import com.netflix.metacat.common.server.usermetadata.TagService; -import com.netflix.metacat.common.server.usermetadata.UserMetadataService; +import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService; +import com.netflix.metacat.common.server.usermetadata.MetadataInterceptor; +import com.netflix.metacat.common.server.usermetadata.MetadataInterceptorImpl; import com.netflix.metacat.common.server.util.DataSourceManager; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -41,7 +43,6 @@ @Configuration @ConditionalOnProperty(value = "metacat.mysqlmetadataservice.enabled", havingValue = "true") public class MySqlUserMetadataConfig { - /** * business Metadata Manager. * @return business Metadata Manager @@ -108,6 +109,20 @@ public TagService tagService( return new MySqlTagService(config, jdbcTemplate, lookupService, metacatJson, userMetadataService); } + /** + * The parentChildRelMetadataService to use. + * + * @param jdbcTemplate JDBC template + * @return The parentChildRelMetadataService implementation backed by MySQL + */ + @Bean + ParentChildRelMetadataService parentChildRelMetadataService( + @Qualifier("metadataJdbcTemplate") final JdbcTemplate jdbcTemplate, + final ConverterUtil converterUtil + ) { + return new MySqlParentChildRelMetaDataService(jdbcTemplate, converterUtil); + } + /** * mySql DataSource. *