Merge branch 'master' into remote-ingest-mcp

.github/workflows/docker-unified.yml

@@ -3,17 +3,9 @@ on:
   push:
     branches:
       - master
-    paths-ignore:
-      - "docs-website/**"
-      - "docs/**"
-      - "**.md"
   pull_request:
     branches:
       - "**"
-    paths-ignore:
-      - "docs-website/**"
-      - "docs/**"
-      - "**.md"
   release:
     types: [published]
 

build.gradle

@@ -73,7 +73,7 @@ plugins {
   id 'com.gorylenko.gradle-git-properties' version '2.4.1'
   id 'com.github.johnrengelman.shadow' version '8.1.1' apply false
   id 'com.palantir.docker' version '0.35.0' apply false
-  id 'com.avast.gradle.docker-compose' version '0.17.5'
+  id 'com.avast.gradle.docker-compose' version '0.17.6'
   id "com.diffplug.spotless" version "6.23.3"
   // https://blog.ltgt.net/javax-jakarta-mess-and-gradle-solution/
   // TODO id "org.gradlex.java-ecosystem-capabilities" version "1.0"
@@ -128,8 +128,8 @@ project.ext.externalDependency = [
     'elasticSearchRest': 'org.opensearch.client:opensearch-rest-high-level-client:' + elasticsearchVersion,
     'elasticSearchJava': 'org.opensearch.client:opensearch-java:2.6.0',
     'findbugsAnnotations': 'com.google.code.findbugs:annotations:3.0.1',
-    'graphqlJava': 'com.graphql-java:graphql-java:19.5',
-    'graphqlJavaScalars': 'com.graphql-java:graphql-java-extended-scalars:19.1',
+    'graphqlJava': 'com.graphql-java:graphql-java:21.3',
+    'graphqlJavaScalars': 'com.graphql-java:graphql-java-extended-scalars:21.0',
     'gson': 'com.google.code.gson:gson:2.8.9',
     'guice': 'com.google.inject:guice:7.0.0',
     'guice4': 'com.google.inject:guice:4.2.3', // Used for frontend while still on old Play version
@@ -221,7 +221,7 @@ project.ext.externalDependency = [
     'resilience4j': 'io.github.resilience4j:resilience4j-retry:1.7.1',
     'rythmEngine': 'org.rythmengine:rythm-engine:1.3.0',
     'servletApi': 'jakarta.servlet:jakarta.servlet-api:6.0.0',
-    'shiroCore': 'org.apache.shiro:shiro-core:1.11.0',
+    'shiroCore': 'org.apache.shiro:shiro-core:1.13.0',
     'snakeYaml': 'org.yaml:snakeyaml:2.0',
     'sparkSql' : 'org.apache.spark:spark-sql_2.12:3.0.3',
     'sparkHive' : 'org.apache.spark:spark-hive_2.12:3.0.3',

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java

@@ -21,6 +21,7 @@ private Constants() {}
   public static final String LINEAGE_SCHEMA_FILE = "lineage.graphql";
   public static final String PROPERTIES_SCHEMA_FILE = "properties.graphql";
   public static final String FORMS_SCHEMA_FILE = "forms.graphql";
+  public static final String INCIDENTS_SCHEMA_FILE = "incident.graphql";
   public static final String BROWSE_PATH_DELIMITER = "/";
   public static final String BROWSE_PATH_V2_DELIMITER = "␟";
   public static final String VERSION_STAMP_FIELD_NAME = "versionStamp";

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -63,6 +63,7 @@
 import com.linkedin.datahub.graphql.generated.GlossaryNode;
 import com.linkedin.datahub.graphql.generated.GlossaryTerm;
 import com.linkedin.datahub.graphql.generated.GlossaryTermAssociation;
+import com.linkedin.datahub.graphql.generated.IncidentSource;
 import com.linkedin.datahub.graphql.generated.IngestionSource;
 import com.linkedin.datahub.graphql.generated.InstitutionalMemoryMetadata;
 import com.linkedin.datahub.graphql.generated.LineageRelationship;
@@ -89,6 +90,7 @@
 import com.linkedin.datahub.graphql.generated.ParentDomainsResult;
 import com.linkedin.datahub.graphql.generated.PolicyMatchCriterionValue;
 import com.linkedin.datahub.graphql.generated.QueryEntity;
+import com.linkedin.datahub.graphql.generated.QueryProperties;
 import com.linkedin.datahub.graphql.generated.QuerySubject;
 import com.linkedin.datahub.graphql.generated.QuickFilter;
 import com.linkedin.datahub.graphql.generated.RecommendationContent;
@@ -125,7 +127,6 @@
 import com.linkedin.datahub.graphql.resolvers.dataproduct.DeleteDataProductResolver;
 import com.linkedin.datahub.graphql.resolvers.dataproduct.ListDataProductAssetsResolver;
 import com.linkedin.datahub.graphql.resolvers.dataproduct.UpdateDataProductResolver;
-import com.linkedin.datahub.graphql.resolvers.dataset.DatasetHealthResolver;
 import com.linkedin.datahub.graphql.resolvers.dataset.DatasetStatsSummaryResolver;
 import com.linkedin.datahub.graphql.resolvers.dataset.DatasetUsageStatsResolver;
 import com.linkedin.datahub.graphql.resolvers.deprecation.UpdateDeprecationResolver;
@@ -140,6 +141,7 @@
 import com.linkedin.datahub.graphql.resolvers.entity.EntityExistsResolver;
 import com.linkedin.datahub.graphql.resolvers.entity.EntityPrivilegesResolver;
 import com.linkedin.datahub.graphql.resolvers.form.BatchAssignFormResolver;
+import com.linkedin.datahub.graphql.resolvers.form.BatchRemoveFormResolver;
 import com.linkedin.datahub.graphql.resolvers.form.CreateDynamicFormAssignmentResolver;
 import com.linkedin.datahub.graphql.resolvers.form.IsFormAssignedToMeResolver;
 import com.linkedin.datahub.graphql.resolvers.form.SubmitFormPromptResolver;
@@ -158,6 +160,10 @@
 import com.linkedin.datahub.graphql.resolvers.group.ListGroupsResolver;
 import com.linkedin.datahub.graphql.resolvers.group.RemoveGroupMembersResolver;
 import com.linkedin.datahub.graphql.resolvers.group.RemoveGroupResolver;
+import com.linkedin.datahub.graphql.resolvers.health.EntityHealthResolver;
+import com.linkedin.datahub.graphql.resolvers.incident.EntityIncidentsResolver;
+import com.linkedin.datahub.graphql.resolvers.incident.RaiseIncidentResolver;
+import com.linkedin.datahub.graphql.resolvers.incident.UpdateIncidentStatusResolver;
 import com.linkedin.datahub.graphql.resolvers.ingest.execution.CancelIngestionExecutionRequestResolver;
 import com.linkedin.datahub.graphql.resolvers.ingest.execution.CreateIngestionExecutionRequestResolver;
 import com.linkedin.datahub.graphql.resolvers.ingest.execution.CreateTestConnectionRequestResolver;
@@ -305,6 +311,7 @@
 import com.linkedin.datahub.graphql.types.form.FormType;
 import com.linkedin.datahub.graphql.types.glossary.GlossaryNodeType;
 import com.linkedin.datahub.graphql.types.glossary.GlossaryTermType;
+import com.linkedin.datahub.graphql.types.incident.IncidentType;
 import com.linkedin.datahub.graphql.types.mlmodel.MLFeatureTableType;
 import com.linkedin.datahub.graphql.types.mlmodel.MLFeatureType;
 import com.linkedin.datahub.graphql.types.mlmodel.MLModelGroupType;
@@ -460,6 +467,7 @@ public class GmsGraphQLEngine {
   private final DataTypeType dataTypeType;
   private final EntityTypeType entityTypeType;
   private final FormType formType;
+  private final IncidentType incidentType;
 
   private final int graphQLQueryComplexityLimit;
   private final int graphQLQueryDepthLimit;
@@ -567,6 +575,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
     this.dataTypeType = new DataTypeType(entityClient);
     this.entityTypeType = new EntityTypeType(entityClient);
     this.formType = new FormType(entityClient);
+    this.incidentType = new IncidentType(entityClient);
 
     this.graphQLQueryComplexityLimit = args.graphQLQueryComplexityLimit;
     this.graphQLQueryDepthLimit = args.graphQLQueryDepthLimit;
@@ -609,7 +618,8 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
             structuredPropertyType,
             dataTypeType,
             entityTypeType,
-            formType);
+            formType,
+            incidentType);
     this.loadableTypes = new ArrayList<>(entityTypes);
     // Extend loadable types with types from the plugins
     // This allows us to offer search and browse capabilities out of the box for those types
@@ -698,6 +708,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) {
     configurePluginResolvers(builder);
     configureStructuredPropertyResolvers(builder);
     configureFormResolvers(builder);
+    configureIncidentResolvers(builder);
   }
 
   private void configureOrganisationRoleResolvers(RuntimeWiring.Builder builder) {
@@ -747,7 +758,8 @@ public GraphQLEngine.Builder builder() {
         .addSchema(fileBasedSchema(STEPS_SCHEMA_FILE))
         .addSchema(fileBasedSchema(LINEAGE_SCHEMA_FILE))
         .addSchema(fileBasedSchema(PROPERTIES_SCHEMA_FILE))
-        .addSchema(fileBasedSchema(FORMS_SCHEMA_FILE));
+        .addSchema(fileBasedSchema(FORMS_SCHEMA_FILE))
+        .addSchema(fileBasedSchema(INCIDENTS_SCHEMA_FILE));
 
     for (GmsGraphQLPlugin plugin : this.graphQLPlugins) {
       List<String> pluginSchemaFiles = plugin.getSchemaFiles();
@@ -1202,7 +1214,12 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) {
                     "createDynamicFormAssignment",
                     new CreateDynamicFormAssignmentResolver(this.formService))
                 .dataFetcher(
-                    "verifyForm", new VerifyFormResolver(this.formService, this.groupService)));
+                    "verifyForm", new VerifyFormResolver(this.formService, this.groupService))
+                .dataFetcher("batchRemoveForm", new BatchRemoveFormResolver(this.formService))
+                .dataFetcher("raiseIncident", new RaiseIncidentResolver(this.entityClient))
+                .dataFetcher(
+                    "updateIncidentStatus",
+                    new UpdateIncidentStatusResolver(this.entityClient, this.entityService)));
   }
 
   private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder) {
@@ -1485,7 +1502,12 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) {
                     .dataFetcher("usageStats", new DatasetUsageStatsResolver(this.usageClient))
                     .dataFetcher("statsSummary", new DatasetStatsSummaryResolver(this.usageClient))
                     .dataFetcher(
-                        "health", new DatasetHealthResolver(graphClient, timeseriesAspectService))
+                        "health",
+                        new EntityHealthResolver(
+                            entityClient,
+                            graphClient,
+                            timeseriesAspectService,
+                            new EntityHealthResolver.Config(true, true)))
                     .dataFetcher("schemaMetadata", new AspectResolver())
                     .dataFetcher(
                         "assertions", new EntityAssertionsResolver(entityClient, graphClient))
@@ -1834,7 +1856,14 @@ private void configureDashboardResolvers(final RuntimeWiring.Builder builder) {
                 .dataFetcher(
                     "statsSummary", new DashboardStatsSummaryResolver(timeseriesAspectService))
                 .dataFetcher("privileges", new EntityPrivilegesResolver(entityClient))
-                .dataFetcher("exists", new EntityExistsResolver(entityService)));
+                .dataFetcher("exists", new EntityExistsResolver(entityService))
+                .dataFetcher(
+                    "health",
+                    new EntityHealthResolver(
+                        entityClient,
+                        graphClient,
+                        timeseriesAspectService,
+                        new EntityHealthResolver.Config(false, true))));
     builder.type(
         "DashboardInfo",
         typeWiring ->
@@ -1951,7 +1980,14 @@ private void configureChartResolvers(final RuntimeWiring.Builder builder) {
                 .dataFetcher(
                     "statsSummary", new ChartStatsSummaryResolver(this.timeseriesAspectService))
                 .dataFetcher("privileges", new EntityPrivilegesResolver(entityClient))
-                .dataFetcher("exists", new EntityExistsResolver(entityService)));
+                .dataFetcher("exists", new EntityExistsResolver(entityService))
+                .dataFetcher(
+                    "health",
+                    new EntityHealthResolver(
+                        entityClient,
+                        graphClient,
+                        timeseriesAspectService,
+                        new EntityHealthResolver.Config(false, true))));
     builder.type(
         "ChartInfo",
         typeWiring ->
@@ -2056,7 +2092,14 @@ private void configureDataJobResolvers(final RuntimeWiring.Builder builder) {
                             }))
                     .dataFetcher("runs", new DataJobRunsResolver(entityClient))
                     .dataFetcher("privileges", new EntityPrivilegesResolver(entityClient))
-                    .dataFetcher("exists", new EntityExistsResolver(entityService)))
+                    .dataFetcher("exists", new EntityExistsResolver(entityService))
+                    .dataFetcher(
+                        "health",
+                        new EntityHealthResolver(
+                            entityClient,
+                            graphClient,
+                            timeseriesAspectService,
+                            new EntityHealthResolver.Config(false, true))))
         .type(
             "DataJobInputOutput",
             typeWiring ->
@@ -2119,7 +2162,14 @@ private void configureDataFlowResolvers(final RuntimeWiring.Builder builder) {
                           return dataFlow.getDataPlatformInstance() != null
                               ? dataFlow.getDataPlatformInstance().getUrn()
                               : null;
-                        })));
+                        }))
+                .dataFetcher(
+                    "health",
+                    new EntityHealthResolver(
+                        entityClient,
+                        graphClient,
+                        timeseriesAspectService,
+                        new EntityHealthResolver.Config(false, true))));
   }
 
   /**
@@ -2536,9 +2586,27 @@ private void configureQueryEntityResolvers(final RuntimeWiring.Builder builder)
     builder
         .type(
             "QueryEntity",
+            typeWiring ->
+                typeWiring
+                    .dataFetcher(
+                        "relationships", new EntityRelationshipsResultResolver(graphClient))
+                    .dataFetcher(
+                        "platform",
+                        new LoadableTypeResolver<>(
+                            dataPlatformType,
+                            (env) -> {
+                              final QueryEntity query = env.getSource();
+                              return query.getPlatform() != null
+                                  ? query.getPlatform().getUrn()
+                                  : null;
+                            })))
+        .type(
+            "QueryProperties",
             typeWiring ->
                 typeWiring.dataFetcher(
-                    "relationships", new EntityRelationshipsResultResolver(graphClient)))
+                    "origin",
+                    new EntityTypeResolver(
+                        entityTypes, (env) -> ((QueryProperties) env.getSource()).getOrigin())))
         .type(
             "ListQueriesResult",
             typeWiring ->
@@ -2660,4 +2728,35 @@ private void configureIngestionSourceResolvers(final RuntimeWiring.Builder build
                               : null;
                         })));
   }
+
+  private void configureIncidentResolvers(final RuntimeWiring.Builder builder) {
+    builder.type(
+        "Incident",
+        typeWiring ->
+            typeWiring.dataFetcher(
+                "relationships", new EntityRelationshipsResultResolver(graphClient)));
+    builder.type(
+        "IncidentSource",
+        typeWiring ->
+            typeWiring.dataFetcher(
+                "source",
+                new LoadableTypeResolver<>(
+                    this.assertionType,
+                    (env) -> {
+                      final IncidentSource incidentSource = env.getSource();
+                      return incidentSource.getSource() != null
+                          ? incidentSource.getSource().getUrn()
+                          : null;
+                    })));
+
+    // Add incidents attribute to all entities that support it
+    final List<String> entitiesWithIncidents =
+        ImmutableList.of("Dataset", "DataJob", "DataFlow", "Dashboard", "Chart");
+    for (String entity : entitiesWithIncidents) {
+      builder.type(
+          entity,
+          typeWiring ->
+              typeWiring.dataFetcher("incidents", new EntityIncidentsResolver(entityClient)));
+    }
+  }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java

@@ -219,5 +219,13 @@ public static boolean isAuthorized(
     return AuthUtil.isAuthorized(authorizer, actor, Optional.of(resourceSpec), privilegeGroup);
   }
 
+  public static boolean isViewDatasetUsageAuthorized(
+      final Urn resourceUrn, final QueryContext context) {
+    return isAuthorized(
+        context,
+        Optional.of(new EntitySpec(resourceUrn.getEntityType(), resourceUrn.toString())),
+        PoliciesConfig.VIEW_DATASET_USAGE_PRIVILEGE);
+  }
+
   private AuthorizationUtils() {}
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/exception/DataHubDataFetcherExceptionHandler.java

@@ -6,6 +6,7 @@
 import graphql.execution.DataFetcherExceptionHandlerResult;
 import graphql.execution.ResultPath;
 import graphql.language.SourceLocation;
+import java.util.concurrent.CompletableFuture;
 import lombok.extern.slf4j.Slf4j;
 
 @PublicApi
@@ -15,7 +16,7 @@ public class DataHubDataFetcherExceptionHandler implements DataFetcherExceptionH
   private static final String DEFAULT_ERROR_MESSAGE = "An unknown error occurred.";
 
   @Override
-  public DataFetcherExceptionHandlerResult onException(
+  public CompletableFuture<DataFetcherExceptionHandlerResult> handleException(
       DataFetcherExceptionHandlerParameters handlerParameters) {
     Throwable exception = handlerParameters.getException();
     SourceLocation sourceLocation = handlerParameters.getSourceLocation();
@@ -44,7 +45,8 @@ public DataFetcherExceptionHandlerResult onException(
       log.error("Failed to execute", exception);
     }
     DataHubGraphQLError error = new DataHubGraphQLError(message, path, sourceLocation, errorCode);
-    return DataFetcherExceptionHandlerResult.newResult().error(error).build();
+    return CompletableFuture.completedFuture(
+        DataFetcherExceptionHandlerResult.newResult().error(error).build());
   }
 
   <T extends Throwable> T findFirstThrowableCauseOfClass(Throwable throwable, Class<T> clazz) {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/ChartStatsSummaryResolver.java

@@ -1,27 +1,20 @@
 package com.linkedin.datahub.graphql.resolvers.chart;
 
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.linkedin.common.urn.Urn;
 import com.linkedin.datahub.graphql.generated.ChartStatsSummary;
 import com.linkedin.metadata.timeseries.TimeseriesAspectService;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 public class ChartStatsSummaryResolver
     implements DataFetcher<CompletableFuture<ChartStatsSummary>> {
 
   private final TimeseriesAspectService timeseriesAspectService;
-  private final Cache<Urn, ChartStatsSummary> summaryCache;
 
   public ChartStatsSummaryResolver(final TimeseriesAspectService timeseriesAspectService) {
     this.timeseriesAspectService = timeseriesAspectService;
-    this.summaryCache =
-        CacheBuilder.newBuilder().maximumSize(10000).expireAfterWrite(6, TimeUnit.HOURS).build();
   }
 
   @Override