Merge branch 'master' into feature/cus-3452-graphql-dataflowjob-conta…

…iner-aspect
datahub-project · Jan 10, 2025 · f20582b · f20582b
2 parents bf35209 + d8e7cb2
commit f20582b
Show file tree

Hide file tree

Showing 29 changed files with 294 additions and 60 deletions.
diff --git a/.github/.codecov.yml b/.github/.codecov.yml
@@ -0,0 +1,65 @@
+comment:
+  layout: "header, files, footer" # remove "new" from "header" and "footer"
+  hide_project_coverage: true # set to false
+  require_changes: false  # if true: only post the comment if coverage changes
+
+codecov:
+  #due to ci-optimization, reports for modules that have not changed may be quite old
+  max_report_age: off
+
+flag_management:
+  default_rules: # the rules that will be followed for any flag added, generally
+    carryforward: true
+    statuses:
+      - type: project
+        target: auto
+        threshold: 0%  #Not enforcing project coverage yet.
+      - type: patch
+        target: 90%
+  individual_flags: # exceptions to the default rules above, stated flag by flag
+    - name: frontend
+      paths:
+        - "datahub-frontend/**"
+        - "datahub-web-react/**"
+    - name: backend
+      paths:
+        - "metadata-models/**"
+        - "datahub-upgrade/**"
+        - "entity-registry/**"
+        - "li-utils/**"
+        - "metadata-auth/**"
+        - "metadata-dao-impl/**"
+        - "metadata-events/**"
+        - "metadata-jobs/**"
+        - "metadata-service/**"
+        - "metadata-utils/**"
+        - "metadata-operation-context/**"
+        - "datahub-graphql-core/**"
+    - name: metadata-io
+      paths:
+        - "metadata-io/**"
+    - name: ingestion
+      paths:
+        - "metadata-ingestion/**"
+    - name: ingestion-airflow
+      paths:
+        - "metadata-ingestion-modules/airflow-plugin/**"
+    - name: ingestion-dagster
+      paths:
+        - "metadata-ingestion-modules/dagster-plugin/**"
+    - name: ingestion-gx-plugin
+      paths:
+        - "metadata-ingestion-modules/gx-plugin/**"
+    - name: ingestion-prefect
+      paths:
+        - "metadata-ingestion-modules/prefect-plugin/**"
+coverage:
+  status:
+    project:
+      default:
+        target: 0% # no threshold enforcement yet
+        only_pulls: true
+    patch:
+      default:
+        target: 90% # for new code added in the patch
+        only_pulls: true
diff --git a/.github/actions/ci-optimization/action.yml b/.github/actions/ci-optimization/action.yml
@@ -13,16 +13,16 @@ outputs:
     value: ${{ steps.filter.outputs.frontend == 'false' && steps.filter.outputs.ingestion == 'false' && steps.filter.outputs.backend == 'true' }}
   backend-change:
     description: "Backend code has changed"
-    value: ${{ steps.filter.outputs.backend == 'true' }}
+    value: ${{ steps.filter.outputs.backend == 'true' ||  steps.trigger.outputs.trigger == 'manual' }}
   ingestion-change:
     description: "Ingestion code has changed"
-    value: ${{ steps.filter.outputs.ingestion == 'true' }}
+    value: ${{ steps.filter.outputs.ingestion == 'true' || steps.trigger.outputs.trigger == 'manual' }}
   ingestion-base-change:
     description: "Ingestion base image docker image has changed"
     value: ${{ steps.filter.outputs.ingestion-base == 'true' }}
   frontend-change:
     description: "Frontend code has changed"
-    value: ${{ steps.filter.outputs.frontend == 'true' }}
+    value: ${{ steps.filter.outputs.frontend == 'true' || steps.trigger.outputs.trigger == 'manual' }}
   docker-change:
     description: "Docker code has changed"
     value: ${{ steps.filter.outputs.docker == 'true' }}
@@ -44,6 +44,15 @@ outputs:
 runs:
   using: "composite"
   steps:
+    - name: Check trigger type
+      id: trigger  # Add an ID to reference this step
+      shell: bash
+      run: |
+        if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+          echo "trigger=manual" >> $GITHUB_OUTPUT
+        else
+          echo "trigger=pr" >> $GITHUB_OUTPUT
+        fi
     - uses: dorny/paths-filter@v3
       id: filter
       with:

diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
@@ -18,6 +18,7 @@ on:
       - "metadata-models/**"
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -77,15 +78,15 @@ jobs:
             **/build/test-results/test/**
             **/junit.*.xml
             !**/binary/**
-      - name: Upload coverage to Codecov
+      - name: Upload coverage to Codecov with ingestion flag
         if: always()
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          directory: ./build/coverage-reports/metadata-ingestion-modules/airflow-plugin/
           fail_ci_if_error: false
-          flags: airflow-${{ matrix.python-version }}-${{ matrix.extra_pip_extras }}
-          name: pytest-airflow
+          flags: ingestion-airflow
+          name: pytest-airflow-${{ matrix.python-version }}-${{ matrix.extra_pip_requirements }}
           verbose: true
       - name: Upload test results to Codecov
         if: ${{ !cancelled() }}

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -12,6 +12,7 @@ on:
     paths-ignore:
       - "docs/**"
       - "**.md"
+  workflow_dispatch:
   release:
     types: [published]
 
@@ -113,25 +114,43 @@ jobs:
         if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
         run: |
           ./gradlew -PjavaClassVersionDefault=8 :metadata-integration:java:spark-lineage:compileJava
+      - name: Gather coverage files
+        run: |
+          echo "BACKEND_FILES=`find ./build/coverage-reports/ -type f | grep -E '(metadata-models|entity-registry|datahuyb-graphql-core|metadata-io|metadata-jobs|metadata-utils|metadata-service|medata-dao-impl|metadata-operation|li-utils|metadata-integration|metadata-events|metadata-auth|ingestion-scheduler|notifications|datahub-upgrade)' | xargs | sed 's/ /,/g'`" >> $GITHUB_ENV
+          echo "FRONTEND_FILES=`find ./build/coverage-reports/ -type f | grep -E '(datahub-frontend|datahub-web-react).*\.(xml|json)$' | xargs | sed 's/ /,/g'`" >> $GITHUB_ENV
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: Test Results (build)
+          name: Test Results (build) - ${{ matrix.command}}-${{ matrix.timezone }}
           path: |
             **/build/reports/tests/test/**
             **/build/test-results/test/**
             **/junit.*.xml
             !**/binary/**
       - name: Ensure codegen is updated
         uses: ./.github/actions/ensure-codegen-updated
-      - name: Upload coverage to Codecov
-        if: always()
+      - name: Upload backend coverage to Codecov
+        if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ${{ env.BACKEND_FILES }}
+          disable_search: true
+          #handle_no_reports_found: true
+          fail_ci_if_error: false
+          flags: backend
+          name: ${{ matrix.command }}
+          verbose: true
+      - name: Upload frontend coverage to Codecov
+        if: ${{  matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }}
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          files: ${{ env.FRONTEND_FILES }}
+          disable_search: true
+          #handle_no_reports_found: true
           fail_ci_if_error: false
-          flags: ${{ matrix.timezone }}
+          flags: frontend
           name: ${{ matrix.command }}
           verbose: true
       - name: Upload test results to Codecov

diff --git a/.github/workflows/dagster-plugin.yml b/.github/workflows/dagster-plugin.yml
@@ -18,6 +18,7 @@ on:
       - "metadata-models/**"
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -64,14 +65,14 @@ jobs:
             **/build/reports/tests/test/**
             **/build/test-results/test/**
             **/junit.*.xml
-      - name: Upload coverage to Codecov
+      - name: Upload coverage to Codecov with ingestion flag
         if: always()
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          directory: ./build/coverage-reports/metadata-ingestion-modules/dagster-plugin/
           fail_ci_if_error: false
-          flags: dagster-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
+          flags: ingestion-dagster-plugin
           name: pytest-dagster
           verbose: true
       - name: Upload test results to Codecov

diff --git a/.github/workflows/gx-plugin.yml b/.github/workflows/gx-plugin.yml
@@ -18,6 +18,7 @@ on:
       - "metadata-models/**"
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -68,14 +69,14 @@ jobs:
             **/build/reports/tests/test/**
             **/build/test-results/test/**
             **/junit.*.xml
-      - name: Upload coverage to Codecov
+      - name: Upload coverage to Codecov with ingestion flag
         if: always()
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          directory: ./build/coverage-reports/metadata-ingestion-modules/gx-plugin/
           fail_ci_if_error: false
-          flags: gx-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
+          flags: ingestion-gx-plugin
           name: pytest-gx
           verbose: true
       - name: Upload test results to Codecov

diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -18,6 +18,7 @@ on:
       - "metadata-models/**"
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -88,15 +89,15 @@ jobs:
             **/build/test-results/test/**
             **/junit.*.xml
             !**/binary/**
-      - name: Upload coverage to Codecov
-        if: ${{ always() }}
+      - name: Upload coverage to Codecov with ingestion flag
+        if: ${{ always() && matrix.python-version == '3.11' }}
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          directory: ./build/coverage-reports/metadata-ingestion/
           fail_ci_if_error: false
-          flags: ingestion-${{ matrix.python-version }}-${{ matrix.command }}
-          name: pytest-ingestion
+          flags: ingestion
+          name: pytest-${{ matrix.python-version }}-${{ matrix.command }}
           verbose: true
       - name: Upload test results to Codecov
         if: ${{ !cancelled() }}

diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml
@@ -20,6 +20,7 @@ on:
       - ".github/workflows/metadata-io.yml"
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -86,8 +87,9 @@ jobs:
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          directory: ./build/coverage-reports/metadata-io/
           fail_ci_if_error: false
+          flags: metadata-io
           name: metadata-io-test
           verbose: true
       - name: Upload test results to Codecov

diff --git a/.github/workflows/prefect-plugin.yml b/.github/workflows/prefect-plugin.yml
@@ -18,6 +18,7 @@ on:
       - "metadata-models/**"
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -60,15 +61,15 @@ jobs:
             **/build/test-results/test/**
             **/junit.*.xml
             !**/binary/**
-      - name: Upload coverage to Codecov
+      - name: Upload coverage to Codecov with ingestion flag
         if: always()
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          directory: ./build/coverage-reports/metadata-ingestion-modules/prefect-plugin/
           fail_ci_if_error: false
-          flags: prefect-${{ matrix.python-version }}
-          name: pytest-prefect
+          flags: ingestion-prefect-plugin
+          name: pytest-prefect-${{ matrix.python-version }}
           verbose: true
       - name: Upload test results to Codecov
         if: ${{ !cancelled() }}

diff --git a/build.gradle b/build.gradle
@@ -286,7 +286,8 @@ project.ext.externalDependency = [
     'annotationApi': 'javax.annotation:javax.annotation-api:1.3.2',
     'jakartaAnnotationApi': 'jakarta.annotation:jakarta.annotation-api:3.0.0',
     'classGraph': 'io.github.classgraph:classgraph:4.8.172',
-    'mustache': 'com.github.spullara.mustache.java:compiler:0.9.14'
+    'mustache': 'com.github.spullara.mustache.java:compiler:0.9.14',
+    'javaxMail': 'com.sun.mail:jakarta.mail:1.6.7'
 ]
 
 allprojects {
@@ -374,6 +375,7 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
     exclude group: "org.slf4j", module: "slf4j-nop"
     exclude group: "org.slf4j", module: "slf4j-ext"
     exclude group: "org.codehaus.jackson", module: "jackson-mapper-asl"
+    exclude group: "javax.mail", module: "mail"
 
     resolutionStrategy.force externalDependency.antlr4Runtime
     resolutionStrategy.force externalDependency.antlr4

diff --git a/docker/datahub-frontend/Dockerfile b/docker/datahub-frontend/Dockerfile
@@ -1,7 +1,7 @@
 # Defining environment
 ARG APP_ENV=prod
 
-FROM alpine:3.20 AS base
+FROM alpine:3.21 AS base
 
 # Configurable repositories
 ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine

diff --git a/docker/datahub-gms/Dockerfile b/docker/datahub-gms/Dockerfile
@@ -23,7 +23,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
 
 RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
 
-FROM alpine:3.20 AS base
+FROM alpine:3.21 AS base
 
 ENV JMX_VERSION=0.18.0
 ENV JETTY_VERSION=11.0.21

diff --git a/docker/datahub-mae-consumer/Dockerfile b/docker/datahub-mae-consumer/Dockerfile
@@ -23,7 +23,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
 
 RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
 
-FROM alpine:3.20 AS base
+FROM alpine:3.21 AS base
 
 # Re-declaring args from above to make them available in this stage (will inherit default values)
 ARG ALPINE_REPO_URL

diff --git a/docker/datahub-mce-consumer/Dockerfile b/docker/datahub-mce-consumer/Dockerfile
@@ -23,7 +23,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
 
 RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
 
-FROM alpine:3.20 AS base
+FROM alpine:3.21 AS base
 
 # Re-declaring args from above to make them available in this stage (will inherit default values)
 ARG ALPINE_REPO_URL

diff --git a/docker/datahub-upgrade/Dockerfile b/docker/datahub-upgrade/Dockerfile
@@ -23,7 +23,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
 
 RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
 
-FROM alpine:3.20 AS base
+FROM alpine:3.21 AS base
 
 # Re-declaring args from above to make them available in this stage (will inherit default values)
 ARG ALPINE_REPO_URL

diff --git a/docker/elasticsearch-setup/Dockerfile b/docker/elasticsearch-setup/Dockerfile
@@ -23,7 +23,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
 
 RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
 
-FROM alpine:3.20 AS base
+FROM alpine:3.21 AS base
 
 ARG ALPINE_REPO_URL
 

diff --git a/docker/mysql-setup/Dockerfile b/docker/mysql-setup/Dockerfile
@@ -17,7 +17,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
 
 RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
 
-FROM alpine:3.20
+FROM alpine:3.21
 COPY --from=binary /go/bin/dockerize /usr/local/bin
 
 ARG ALPINE_REPO_URL