[Feature][Server] Add metadata lineage api

datavines-core/src/main/java/io/datavines/core/enums/Status.java

@@ -129,8 +129,9 @@ public enum Status {
     CREATE_CATALOG_TASK_SCHEDULE_ERROR(20030002, "Create Catalog Task Schedule {0} Error", "创建元数据抓取定时任务 {0} 错误"),
     CATALOG_TASK_SCHEDULE_NOT_EXIST_ERROR(20030003, "Catalog Task Schedule {0} is not Exist error", "元数据抓取定时任务 {0} 不存在"),
     UPDATE_CATALOG_TASK_SCHEDULE_ERROR(20030004, "Update Catalog Task Schedule {0} Error", "更新元数据抓取定时任务 {0} 错误"),
-    CATALOG_PROFILE_INSTANCE_FQN_ERROR(20030004, "Catalog instance fqn {0} Error", "数据实体全限定名 {0} 错误"),
+    CATALOG_PROFILE_INSTANCE_FQN_ERROR(20030005, "Catalog instance fqn {0} Error", "数据实体全限定名 {0} 错误"),
 
+    CATALOG_PROFILE_INSTANCE_IS_NULL_ERROR(20040001, "Catalog instance fqn {0} Error", "数据实体 {0} 为空错误"),
     CREATE_ISSUE_ERROR(21010001, "Create Issue {0} Error", "创建Issue {0} 错误"),
     ISSUE_NOT_EXIST_ERROR(21010002, "Issue {0} Not Exist Error", "Issue {0} 不存在错误"),
     ISSUE_EXIST_ERROR(21010003, "Issue {0} is Exist error", "Issue {0} 已存在错误"),

datavines-server/src/main/java/io/datavines/server/api/controller/CatalogLineageController.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.server.api.controller;
+
+import io.datavines.core.aop.RefreshToken;
+import io.datavines.core.constant.DataVinesConstants;
+import io.datavines.server.api.dto.bo.catalog.lineage.EntityEdgeInfo;
+import io.datavines.server.repository.entity.catalog.CatalogTagCategory;
+import io.datavines.server.repository.service.CatalogEntityRelService;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.*;
+
+import javax.validation.Valid;
+
+@Api(value = "catalog", tags = "catalog", produces = MediaType.APPLICATION_JSON_VALUE)
+@RestController
+@RequestMapping(value = DataVinesConstants.BASE_API_PATH + "/catalog/lineage", produces = MediaType.APPLICATION_JSON_VALUE)
+@RefreshToken
+public class CatalogLineageController {
+
+    @Autowired
+    private CatalogEntityRelService catalogEntityRelService;
+
+    @ApiOperation(value = "add lineage", response = Long.class)
+    @PostMapping(value = "/add", consumes = MediaType.APPLICATION_JSON_VALUE)
+    public Object addLineage(@Valid @RequestBody EntityEdgeInfo entityEdgeInfo) {
+        return catalogEntityRelService.addLineage(entityEdgeInfo);
+    }
+
+    @ApiOperation(value = "get lineage by full qualified name", response = CatalogTagCategory.class, responseContainer = "list")
+    @GetMapping(value = "/getByFqn/{datasourceId}/{fqn}")
+    public Object getByFqn(@PathVariable Long datasourceId, @PathVariable String fqn) {
+        return catalogEntityRelService.getLineageByFqn(datasourceId,fqn,1,1);
+    }
+
+    @ApiOperation(value = "delete tag category", response = boolean.class)
+    @GetMapping(value = "/getByUUID/{uuid}")
+    public Object getByUUID(@PathVariable String uuid) {
+        return catalogEntityRelService.getLineageByUUID(uuid,1,1);
+    }
+
+    @ApiOperation(value = "delete lineage", response = boolean.class)
+    @DeleteMapping(value = "/{fromUUID}/{toUUID}")
+    public Object deleteLineage(@PathVariable("fromUUID") String fromUUID,
+                                     @PathVariable("toUUID") String toUUID) {
+        return catalogEntityRelService.deleteLineage(fromUUID, toUUID);
+    }
+}

datavines-server/src/main/java/io/datavines/server/api/dto/bo/catalog/CatalogEntityInstanceInfo.java

@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.server.api.dto.bo.catalog;
+
+import lombok.Data;
+
+import java.io.Serializable;
+
+@Data
+public class CatalogEntityInstanceInfo implements Serializable {
+
+    private static final long serialVersionUID = -1L;
+
+    private Long id;
+
+    private String uuid;
+
+    private Long datasourceId;
+
+    private String type;
+
+    private String fullyQualifiedName;
+
+    private String displayName;
+
+    private String description;
+
+}

datavines-server/src/main/java/io/datavines/server/api/dto/bo/catalog/lineage/CatalogEntityColumnLineageDetail.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.server.api.dto.bo.catalog.lineage;
+
+import lombok.Data;
+
+import java.io.Serializable;
+import java.util.List;
+
+@Data
+public class CatalogEntityColumnLineageDetail implements Serializable {
+
+    private static final long serialVersionUID = -1L;
+
+    private List<String> fromChildren;
+
+    private String function;
+
+    private String toChild;
+}

datavines-server/src/main/java/io/datavines/server/api/dto/bo/catalog/lineage/CatalogEntityLineageDetail.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.server.api.dto.bo.catalog.lineage;
+
+import io.datavines.server.enums.LineageSourceType;
+import lombok.Data;
+
+import java.io.Serializable;
+
+@Data
+public class CatalogEntityLineageDetail implements Serializable {
+
+    private static final long serialVersionUID = -1L;
+
+    private CatalogEntityColumnLineageDetail childRelDetail;
+
+    private String description;
+
+    private LineageSourceType sourceType;
+
+    private String sqlQuery;
+}

datavines-server/src/main/java/io/datavines/server/api/dto/bo/catalog/lineage/EntityEdgeInfo.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.server.api.dto.bo.catalog.lineage;
+
+import io.datavines.server.api.dto.bo.catalog.CatalogEntityInstanceInfo;
+import lombok.Data;
+
+import java.io.Serializable;
+
+@Data
+public class EntityEdgeInfo implements Serializable {
+
+    private static final long serialVersionUID = -1L;
+
+    private CatalogEntityInstanceInfo fromEntity;
+
+    private String description;
+
+    private CatalogEntityLineageDetail lineageDetail;
+
+    private CatalogEntityInstanceInfo toEntity;
+}

datavines-server/src/main/java/io/datavines/server/api/dto/vo/catalog/lineage/CatalogEntityLineageVO.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.server.api.dto.vo.catalog.lineage;
+
+import io.datavines.server.api.dto.bo.catalog.CatalogEntityInstanceInfo;
+import io.datavines.server.api.dto.bo.catalog.lineage.EntityEdgeInfo;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.util.List;
+
+@Data
+public class CatalogEntityLineageVO implements Serializable {
+
+    public List<EntityEdgeInfo> upstreamEdges;
+
+    public List<EntityEdgeInfo> downstreamEdges;
+
+    public CatalogEntityInstanceInfo currentEntity;
+}

datavines-server/src/main/java/io/datavines/server/enums/LineageSourceType.java

@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.server.enums;
+
+import com.baomidou.mybatisplus.annotation.EnumValue;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public enum LineageSourceType {
+
+    /**
+     * 0-data quality task
+     * 1-catalog task
+     */
+    MANUAL(0, "manual"),
+    SQL_PARSER(1, "sql_parser"),
+    SPARK_LISTENER(2, "spark_listener"),
+    FLINK_SQL_LINEAGE(3, "flink_sql_lineage"),
+    ;
+    LineageSourceType(int code, String description){
+        this.code = code;
+        this.description = description;
+    }
+
+    @EnumValue
+    private final int code;
+
+    private final String description;
+
+    private static final Map<Integer, LineageSourceType> COMMAND_CATEGORY_MAP = new HashMap<>();
+
+    static {
+        for (LineageSourceType commandCategory : LineageSourceType.values()) {
+            COMMAND_CATEGORY_MAP.put(commandCategory.code,commandCategory);
+        }
+    }
+
+    public static LineageSourceType of(Integer status) {
+        if (COMMAND_CATEGORY_MAP.containsKey(status)) {
+            return COMMAND_CATEGORY_MAP.get(status);
+        }
+        throw new IllegalArgumentException("invalid command category : " + status);
+    }
+
+    public int getCode() {
+        return code;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+}

datavines-server/src/main/java/io/datavines/server/repository/entity/catalog/CatalogEntityRel.java

@@ -44,6 +44,12 @@ public class CatalogEntityRel implements Serializable {
     @TableField(value = "type")
     private String type;
 
+    @TableField(value = "source_type")
+    private String sourceType;
+
+    @TableField(value = "related_script")
+    private String relatedScript;
+
     @TableField(value = "update_by")
     private Long updateBy;
 

datavines-server/src/main/java/io/datavines/server/repository/service/CatalogEntityInstanceService.java

@@ -34,6 +34,8 @@ public interface CatalogEntityInstanceService extends IService<CatalogEntityInst
 
     CatalogEntityInstance getByTypeAndFQN(String type, String fqn);
 
+    CatalogEntityInstance getByUUID(String uuid);
+
     CatalogEntityInstance getByDataSourceAndFQN(Long dataSourceId, String fqn);
 
     IPage<CatalogEntityInstance> getEntityPage(String upstreamId, Integer pageNumber, Integer pageSize, String whetherMark);

datavines-server/src/main/java/io/datavines/server/repository/service/CatalogEntityRelService.java

@@ -17,7 +17,17 @@
 package io.datavines.server.repository.service;
 
 import com.baomidou.mybatisplus.extension.service.IService;
+import io.datavines.server.api.dto.bo.catalog.lineage.EntityEdgeInfo;
+import io.datavines.server.api.dto.vo.catalog.lineage.CatalogEntityLineageVO;
 import io.datavines.server.repository.entity.catalog.CatalogEntityRel;
 
 public interface CatalogEntityRelService extends IService<CatalogEntityRel> {
+
+    public boolean addLineage(EntityEdgeInfo entityEdgeInfo);
+
+    public CatalogEntityLineageVO getLineageByFqn(Long datasourceId, String fqn, int upstreamDepth, int downstreamDepth);
+
+    public CatalogEntityLineageVO getLineageByUUID(String uuid, int upstreamDepth, int downstreamDepth);
+
+    public boolean deleteLineage(String fromUUID, String toUUID);
 }

datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogEntityInstanceServiceImpl.java

@@ -94,6 +94,11 @@ public CatalogEntityInstance getByTypeAndFQN(String type, String fqn) {
         return baseMapper.selectOne(new QueryWrapper<CatalogEntityInstance>().lambda().eq(CatalogEntityInstance::getType, type).eq(CatalogEntityInstance::getFullyQualifiedName, fqn));
     }
 
+    @Override
+    public CatalogEntityInstance getByUUID(String uuid) {
+        return baseMapper.selectOne(new QueryWrapper<CatalogEntityInstance>().lambda().eq(CatalogEntityInstance::getUuid, uuid));
+    }
+
     @Override
     public CatalogEntityInstance getByDataSourceAndFQN(Long dataSourceId, String fqn) {
         return baseMapper.selectOne(new QueryWrapper<CatalogEntityInstance>().lambda()