Skip to content

Commit

Permalink
adding dataTransformLogic models
Browse files Browse the repository at this point in the history
  • Loading branch information
gabe-lyons committed Dec 20, 2024
1 parent acb76cd commit c6fd3bc
Show file tree
Hide file tree
Showing 12 changed files with 424 additions and 58 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package com.linkedin.datahub.graphql.types.common.mappers;

import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.DataTransform;
import com.linkedin.datahub.graphql.generated.DataTransformLogic;
import com.linkedin.datahub.graphql.generated.QueryLanguage;
import com.linkedin.datahub.graphql.generated.QueryStatement;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;

public class DataTransformLogicMapper
implements ModelMapper<
com.linkedin.common.DataTransformLogic,
com.linkedin.datahub.graphql.generated.DataTransformLogic> {

public static final DataTransformLogicMapper INSTANCE = new DataTransformLogicMapper();

public static DataTransformLogic map(
@Nullable final QueryContext context,
@Nonnull final com.linkedin.common.DataTransformLogic input) {
return INSTANCE.apply(context, input);
}

@Override
public DataTransformLogic apply(
@Nullable final QueryContext context,
@Nonnull final com.linkedin.common.DataTransformLogic input) {

final DataTransformLogic result = new DataTransformLogic();

// Map transforms array using DataTransformMapper
result.setTransforms(
input.getTransforms().stream()
.map(transform -> DataTransformMapper.map(context, transform))
.collect(Collectors.toList()));

return result;
}
}

class DataTransformMapper
implements ModelMapper<
com.linkedin.common.DataTransform, com.linkedin.datahub.graphql.generated.DataTransform> {

public static final DataTransformMapper INSTANCE = new DataTransformMapper();

public static DataTransform map(
@Nullable final QueryContext context,
@Nonnull final com.linkedin.common.DataTransform input) {
return INSTANCE.apply(context, input);
}

@Override
public DataTransform apply(
@Nullable final QueryContext context,
@Nonnull final com.linkedin.common.DataTransform input) {

final DataTransform result = new DataTransform();

// Map query statement if present
if (input.hasQueryStatement()) {
QueryStatement statement =
new QueryStatement(
input.getQueryStatement().getValue(),
QueryLanguage.valueOf(input.getQueryStatement().getLanguage().toString()));
result.setQueryStatement(statement);
}

return result;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package com.linkedin.datahub.graphql.types.common.mappers;

import com.linkedin.data.template.GetMode;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.*;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
import com.linkedin.query.QueryProperties;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;

public class QueryPropertiesMapper
implements ModelMapper<
QueryProperties, com.linkedin.datahub.graphql.generated.QueryProperties> {

public static final QueryPropertiesMapper INSTANCE = new QueryPropertiesMapper();

public static com.linkedin.datahub.graphql.generated.QueryProperties map(
@Nullable final QueryContext context, @Nonnull final QueryProperties input) {
return INSTANCE.apply(context, input);
}

@Override
public com.linkedin.datahub.graphql.generated.QueryProperties apply(
@Nullable final QueryContext context, @Nonnull final QueryProperties input) {

final com.linkedin.datahub.graphql.generated.QueryProperties result =
new com.linkedin.datahub.graphql.generated.QueryProperties();

// Map Query Source
result.setSource(QuerySource.valueOf(input.getSource().toString()));

// Map Query Statement
result.setStatement(
new QueryStatement(
input.getStatement().getValue(),
QueryLanguage.valueOf(input.getStatement().getLanguage().toString())));

// Map optional fields
result.setName(input.getName(GetMode.NULL));
result.setDescription(input.getDescription(GetMode.NULL));

// Map origin if present
if (input.hasOrigin() && input.getOrigin() != null) {
result.setOrigin(UrnToEntityMapper.map(context, input.getOrigin()));
}

// Map created audit stamp
AuditStamp created = new AuditStamp();
created.setTime(input.getCreated().getTime());
created.setActor(input.getCreated().getActor(GetMode.NULL).toString());
result.setCreated(created);

// Map last modified audit stamp
AuditStamp lastModified = new AuditStamp();
lastModified.setTime(input.getLastModified().getTime());
lastModified.setActor(input.getLastModified().getActor(GetMode.NULL).toString());
result.setLastModified(lastModified);

return result;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,8 @@ public class DataJobType
BROWSE_PATHS_V2_ASPECT_NAME,
SUB_TYPES_ASPECT_NAME,
STRUCTURED_PROPERTIES_ASPECT_NAME,
FORMS_ASPECT_NAME);
FORMS_ASPECT_NAME,
DATA_TRANSFORM_LOGIC_ASPECT_NAME);
private static final Set<String> FACET_FIELDS = ImmutableSet.of("flow");
private final EntityClient _entityClient;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,7 @@
import static com.linkedin.metadata.Constants.*;

import com.google.common.collect.ImmutableList;
import com.linkedin.common.BrowsePathsV2;
import com.linkedin.common.DataPlatformInstance;
import com.linkedin.common.Deprecation;
import com.linkedin.common.Forms;
import com.linkedin.common.GlobalTags;
import com.linkedin.common.GlossaryTerms;
import com.linkedin.common.InstitutionalMemory;
import com.linkedin.common.Ownership;
import com.linkedin.common.Status;
import com.linkedin.common.SubTypes;
import com.linkedin.common.*;
import com.linkedin.common.urn.Urn;
import com.linkedin.data.DataMap;
import com.linkedin.datahub.graphql.QueryContext;
Expand All @@ -26,15 +17,7 @@
import com.linkedin.datahub.graphql.generated.DataJobProperties;
import com.linkedin.datahub.graphql.generated.Dataset;
import com.linkedin.datahub.graphql.generated.EntityType;
import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper;
import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper;
import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper;
import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper;
import com.linkedin.datahub.graphql.types.common.mappers.FineGrainedLineagesMapper;
import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper;
import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper;
import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper;
import com.linkedin.datahub.graphql.types.common.mappers.SubTypesMapper;
import com.linkedin.datahub.graphql.types.common.mappers.*;
import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
import com.linkedin.datahub.graphql.types.form.FormsMapper;
Expand Down Expand Up @@ -139,6 +122,9 @@ public DataJob apply(
context, new StructuredProperties(data), entityUrn));
} else if (FORMS_ASPECT_NAME.equals(name)) {
result.setForms(FormsMapper.map(new Forms(data), entityUrn.toString()));
} else if (DATA_TRANSFORM_LOGIC_ASPECT_NAME.equals(name)) {
result.setDataTransformLogic(
DataTransformLogicMapper.map(context, new DataTransformLogic(data)));
}
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,13 @@
import com.linkedin.common.DataPlatformInstance;
import com.linkedin.common.urn.Urn;
import com.linkedin.data.DataMap;
import com.linkedin.data.template.GetMode;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.AuditStamp;
import com.linkedin.datahub.graphql.generated.DataPlatform;
import com.linkedin.datahub.graphql.generated.Dataset;
import com.linkedin.datahub.graphql.generated.EntityType;
import com.linkedin.datahub.graphql.generated.QueryEntity;
import com.linkedin.datahub.graphql.generated.QueryLanguage;
import com.linkedin.datahub.graphql.generated.QuerySource;
import com.linkedin.datahub.graphql.generated.QueryStatement;
import com.linkedin.datahub.graphql.generated.QuerySubject;
import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
import com.linkedin.datahub.graphql.types.common.mappers.QueryPropertiesMapper;
import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
import com.linkedin.entity.EntityResponse;
Expand Down Expand Up @@ -48,7 +43,10 @@ public QueryEntity apply(
result.setType(EntityType.QUERY);
EnvelopedAspectMap aspectMap = entityResponse.getAspects();
MappingHelper<QueryEntity> mappingHelper = new MappingHelper<>(aspectMap, result);
mappingHelper.mapToResult(context, QUERY_PROPERTIES_ASPECT_NAME, this::mapQueryProperties);
mappingHelper.mapToResult(
QUERY_PROPERTIES_ASPECT_NAME,
(entity, dataMap) ->
entity.setProperties(QueryPropertiesMapper.map(context, new QueryProperties(dataMap))));
mappingHelper.mapToResult(QUERY_SUBJECTS_ASPECT_NAME, this::mapQuerySubjects);
mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, this::mapPlatform);
return mappingHelper.getResult();
Expand All @@ -64,37 +62,6 @@ private void mapPlatform(@Nonnull QueryEntity query, @Nonnull DataMap dataMap) {
}
}

private void mapQueryProperties(
@Nullable final QueryContext context, @Nonnull QueryEntity query, @Nonnull DataMap dataMap) {
QueryProperties queryProperties = new QueryProperties(dataMap);
com.linkedin.datahub.graphql.generated.QueryProperties res =
new com.linkedin.datahub.graphql.generated.QueryProperties();

// Query Source must be kept in sync.
res.setSource(QuerySource.valueOf(queryProperties.getSource().toString()));
res.setStatement(
new QueryStatement(
queryProperties.getStatement().getValue(),
QueryLanguage.valueOf(queryProperties.getStatement().getLanguage().toString())));
res.setName(queryProperties.getName(GetMode.NULL));
res.setDescription(queryProperties.getDescription(GetMode.NULL));
if (queryProperties.hasOrigin() && queryProperties.getOrigin() != null) {
res.setOrigin(UrnToEntityMapper.map(context, queryProperties.getOrigin()));
}

AuditStamp created = new AuditStamp();
created.setTime(queryProperties.getCreated().getTime());
created.setActor(queryProperties.getCreated().getActor(GetMode.NULL).toString());
res.setCreated(created);

AuditStamp lastModified = new AuditStamp();
lastModified.setTime(queryProperties.getLastModified().getTime());
lastModified.setActor(queryProperties.getLastModified().getActor(GetMode.NULL).toString());
res.setLastModified(lastModified);

query.setProperties(res);
}

@Nonnull
private void mapQuerySubjects(@Nonnull QueryEntity query, @Nonnull DataMap dataMap) {
QuerySubjects querySubjects = new QuerySubjects(dataMap);
Expand Down
25 changes: 25 additions & 0 deletions datahub-graphql-core/src/main/resources/entity.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -6556,6 +6556,11 @@ type DataJob implements EntityWithRelationships & Entity & BrowsableEntity {
The forms associated with the Dataset
"""
forms: Forms

"""
Data Transform Logic associated with the Data Job
"""
dataTransformLogic: DataTransformLogic
}

"""
Expand Down Expand Up @@ -6773,6 +6778,26 @@ type DataJobInputOutput {
fineGrainedLineages: [FineGrainedLineage!]
}

"""
Information about a transformation applied to data assets
"""
type DataTransform {
"""
The transformation may be defined by a query statement
"""
queryStatement: QueryStatement
}

"""
Information about transformations applied to data assets
"""
type DataTransformLogic {
"""
List of transformations applied
"""
transforms: [DataTransform!]!
}

"""
Information about individual user usage of a Dataset
"""
Expand Down
Loading

0 comments on commit c6fd3bc

Please sign in to comment.