GDP-2719 Update to 0.13.1

.dockerignore

@@ -1,7 +1,11 @@
 **/node_modules/
 */build/
 */*/build/
-*/venv/
+**/venv/
+**/.tox/
+**/.mypy_cache/
+**/.pytest_cache/
+**/__pycache__/
 out
 **/*.class
 # Have to copy gradle/wrapper/gradle-wrapper.jar, can't exclude ALL jars

.drone.yml

@@ -3,7 +3,7 @@ name: default
 
 anchors:
   build_frontend: &build_frontend
-    image: gradle:jdk11
+    image: gradle:jdk17
     commands:
       - cd "/drone/src/$${ENVIRONMENT}"
       - sed -i "s/<AMPLITUDE_API_KEY>/$AMPLITUDE_API_KEY/g" datahub-web-react/src/conf/analytics.ts

.github/actions/ci-optimization/action.yml

@@ -0,0 +1,87 @@
+name: 'Identify CI Optimizations'
+description: 'Determine if code changes are specific to certain modules.'
+
+outputs:
+  frontend-only:
+    description: "Frontend only change"
+    value: ${{ steps.filter.outputs.frontend == 'true' && steps.filter.outputs.ingestion == 'false' && steps.filter.outputs.backend == 'false' }}
+  ingestion-only:
+    description: "Ingestion only change"
+    value: ${{ steps.filter.outputs.frontend == 'false' && steps.filter.outputs.ingestion == 'true' && steps.filter.outputs.backend == 'false' }}
+  backend-only:
+    description: "Backend only change"
+    value: ${{ steps.filter.outputs.frontend == 'false' && steps.filter.outputs.ingestion == 'false' && steps.filter.outputs.backend == 'true' }}
+  backend-change:
+    description: "Backend code has changed"
+    value: ${{ steps.filter.outputs.backend == 'true' }}
+  ingestion-change:
+    description: "Ingestion code has changed"
+    value: ${{ steps.filter.outputs.ingestion == 'true' }}
+  frontend-change:
+    description: "Frontend code has changed"
+    value: ${{ steps.filter.outputs.frontend == 'true' }}
+  docker-change:
+    description: "Docker code has changed"
+    value: ${{ steps.filter.outputs.docker == 'true' }}
+  kafka-setup-change:
+    description: "Kafka setup docker change"
+    value: ${{ steps.filter.outputs.kafka-setup == 'true' }}
+  mysql-setup-change:
+    description: "Mysql setup docker change"
+    value: ${{ steps.filter.outputs.mysql-setup == 'true' }}
+  postgres-setup-change:
+    description: "Postgres setup docker change"
+    value: ${{ steps.filter.outputs.postgres-setup == 'true' }}
+  elasticsearch-setup-change:
+    description: "Elasticsearch setup docker change"
+    value: ${{ steps.filter.outputs.elasticsearch-setup == 'true' }}
+  smoke-test-change:
+    description: "Smoke test change"
+    value: ${{ steps.filter.outputs.smoke-test == 'true' }}
+runs:
+  using: "composite"
+  steps:
+    - uses: dorny/paths-filter@v2
+      id: filter
+      with:
+        filters: |
+          frontend:
+            - "datahub-frontend/**"
+            - "datahub-web-react/**"
+            - "smoke-test/tests/cypress/**"
+            - "docker/datahub-frontend/**"
+          ingestion:
+            - "metadata-ingestion-modules/**"
+            - "metadata-ingestion/**"
+            - "metadata-models/**"
+            - "smoke-test/**"
+            - "docker/datahub-ingestion**"
+          docker:
+            - "docker/**"
+          backend:
+            - ".github/**"
+            - "metadata-models/**"
+            - "datahub-upgrade/**"
+            - "entity-registry/**"
+            - "li-utils/**"
+            - "metadata-auth/**"
+            - "metadata-dao-impl/**"
+            - "metadata-events/**"
+            - "metadata-io/**"
+            - "metadata-jobs/**"
+            - "metadata-service/**"
+            - "metadata-utils/**"
+            - "metadata-operation-context/**"
+            - "datahub-graphql-core/**"
+            - "smoke-test/**"
+            - "docker/**"
+          kafka-setup:
+            - "docker/kafka-setup/**"
+          mysql-setup:
+            - "docker/mysql-setup/**"
+          postgres-setup:
+            - "docker/postgres-setup/**"
+          elasticsearch-setup:
+            - "docker/elasticsearch-setup/**"
+          smoke-test:
+            - "smoke-test/**"

.github/actions/docker-custom-build-and-push/action.yml

@@ -20,7 +20,7 @@ inputs:
     required: false
 
   images:
-    # e.g. linkedin/datahub-gms
+    # e.g. acryldata/datahub-gms
     description: "List of Docker images to use as base name for tags"
     required: true
   build-args:

.github/scripts/check_python_package.py

@@ -0,0 +1,18 @@
+import setuptools
+
+folders = ["./smoke-test/tests"]
+
+for folder in folders:
+    print(f"Checking folder {folder}")
+    a = [i for i in setuptools.find_packages(folder) if "cypress" not in i]
+    b = [i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i]
+
+    in_a_not_b = set(a) - set(b)
+    in_b_not_a = set(b) - set(a)
+
+    assert (
+        len(in_a_not_b) == 0
+    ), f"Found packages in {folder} that are not in namespace packages: {in_a_not_b}"
+    assert (
+        len(in_b_not_a) == 0
+    ), f"Found namespace packages in {folder} that are not in packages: {in_b_not_a}"

.github/scripts/docker_helpers.sh

@@ -24,7 +24,7 @@ function get_tag_full {
 }
 
 function get_python_docker_release_v {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},0.0.0+docker.${SHORT_SHA},g" -e 's,refs/tags/v\(.*\),\1+docker,g' -e 's,refs/pull/\([0-9]*\).*,0.0.0+docker.pr\1,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},1!0.0.0+docker.${SHORT_SHA},g" -e 's,refs/tags/v\(.*\),1!\1+docker,g' -e 's,refs/pull/\([0-9]*\).*,1!0.0.0+docker.pr\1,g')
 }
 
 function get_unique_tag {
@@ -37,4 +37,4 @@ function get_unique_tag_slim {
 
 function get_unique_tag_full {
     echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
-}
+}

.github/workflows/airflow-plugin.yml

@@ -32,32 +32,42 @@ jobs:
     strategy:
       matrix:
         include:
+          # Note: this should be kept in sync with tox.ini.
           - python-version: "3.8"
             extra_pip_requirements: "apache-airflow~=2.1.4"
             extra_pip_extras: plugin-v1
           - python-version: "3.8"
             extra_pip_requirements: "apache-airflow~=2.2.4"
             extra_pip_extras: plugin-v1
           - python-version: "3.10"
-            extra_pip_requirements: "apache-airflow~=2.4.0"
+            extra_pip_requirements: "apache-airflow==2.4.3"
+            extra_pip_extras: plugin-v2,test-airflow24
+          - python-version: "3.10"
+            extra_pip_requirements: 'apache-airflow==2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt'
             extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extra_pip_requirements: "apache-airflow~=2.6.0"
+            extra_pip_requirements: 'apache-airflow==2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt'
             extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extra_pip_requirements: "apache-airflow>=2.7.0"
+            extra_pip_requirements: 'apache-airflow==2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt'
             extra_pip_extras: plugin-v2
       fail-fast: false
     steps:
-      - uses: actions/checkout@v3
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: 17
+      - uses: gradle/gradle-build-action@v2
+      - uses: acryldata/sane-checkout-action@v3
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Install airflow package and test (extras ${{ matrix.extra_pip_requirements }})
-        run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:lint :metadata-ingestion-modules:airflow-plugin:testQuick
+        run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build
       - name: pip freeze show list installed
         if: always()
         run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && pip freeze
@@ -69,15 +79,16 @@ jobs:
             **/build/reports/tests/test/**
             **/build/test-results/test/**
             **/junit.*.xml
+            !**/binary/**
       - name: Upload coverage to Codecov
         if: always()
         uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           directory: .
           fail_ci_if_error: false
-          flags: airflow-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
-          name: pytest-airflow
+          flags: airflow,airflow-${{ matrix.extra_pip_extras }}
+          name: pytest-airflow-${{ matrix.python-version }}-${{ matrix.extra_pip_requirements }}
           verbose: true
 
   event-file:

.github/workflows/build-and-test.yml

@@ -20,6 +20,25 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      frontend_change: ${{ steps.ci-optimize.outputs.frontend-change == 'true' }}
+      ingestion_change: ${{ steps.ci-optimize.outputs.ingestion-change == 'true' }}
+      backend_change: ${{ steps.ci-optimize.outputs.backend-change == 'true' }}
+      docker_change: ${{ steps.ci-optimize.outputs.docker-change == 'true' }}
+      frontend_only: ${{ steps.ci-optimize.outputs.frontend-only == 'true' }}
+      ingestion_only: ${{ steps.ci-optimize.outputs.ingestion-only == 'true' }}
+      kafka_setup_change: ${{ steps.ci-optimize.outputs.kafka-setup-change == 'true' }}
+      mysql_setup_change: ${{ steps.ci-optimize.outputs.mysql-setup-change == 'true' }}
+      postgres_setup_change: ${{ steps.ci-optimize.outputs.postgres-setup-change == 'true' }}
+      elasticsearch_setup_change: ${{ steps.ci-optimize.outputs.elasticsearch-setup-change == 'true' }}
+    steps:
+      - name: Check out the repo
+        uses: acryldata/sane-checkout-action@v3
+      - uses: ./.github/actions/ci-optimization
+        id: ci-optimize
+
   build:
     strategy:
       fail-fast: false
@@ -29,34 +48,57 @@ jobs:
             "except_metadata_ingestion",
             "frontend",
           ]
-        timezone: ["UTC", "America/New_York"]
+        timezone: ["UTC"]
+        include:
+          # We only need the timezone variation for frontend tests.
+          - command: "frontend"
+            timezone: "America/New_York"
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    needs: setup
     steps:
-      - uses: szenius/set-timezone@v1.0
+      - uses: szenius/set-timezone@v1.2
         with:
           timezoneLinux: ${{ matrix.timezone }}
-      - uses: hsheth2/sane-checkout-action@v1
-      - name: Set up JDK 11
+      - name: Check out the repo
+        uses: acryldata/sane-checkout-action@v3
+      - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
-          java-version: 11
+          java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - uses: actions/setup-python@v4
+        if: ${{ needs.setup.outputs.ingestion_change == 'true' }}
         with:
           python-version: "3.10"
           cache: pip
-      - name: Gradle build (and test) for metadata ingestion
-        # we only need the timezone runs for frontend tests
-        if: ${{  matrix.command == 'except_metadata_ingestion' && matrix.timezone == 'America/New_York' }}
+      - name: Gradle build (and test) for NOT metadata ingestion
+        if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
         run: |
-          ./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test -x :metadata-ingestion-modules:airflow-plugin:build -x :metadata-ingestion-modules:airflow-plugin:check -x :datahub-frontend:build -x :datahub-web-react:build --parallel
+          ./gradlew build \
+            -x :metadata-ingestion:build \
+            -x :metadata-ingestion:check \
+            -x :docs-website:build \
+            -x :metadata-integration:java:spark-lineage:test \
+            -x :metadata-io:test \
+            -x :metadata-ingestion-modules:airflow-plugin:build \
+            -x :metadata-ingestion-modules:airflow-plugin:check \
+            -x :metadata-ingestion-modules:dagster-plugin:build \
+            -x :metadata-ingestion-modules:dagster-plugin:check \
+            -x :datahub-frontend:build \
+            -x :datahub-web-react:build \
+            --parallel
       - name: Gradle build (and test) for frontend
-        if: ${{  matrix.command == 'frontend' }}
+        if: ${{  matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }}
         run: |
           ./gradlew :datahub-frontend:build :datahub-web-react:build --parallel
         env:
           NODE_OPTIONS: "--max-old-space-size=3072"
+      - name: Gradle compile (jdk8) for legacy Spark
+        if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
+        run: |
+          ./gradlew -PjavaClassVersionDefault=8 :metadata-integration:java:spark-lineage:compileJava
       - uses: actions/upload-artifact@v3
         if: always()
         with:
@@ -65,27 +107,20 @@ jobs:
             **/build/reports/tests/test/**
             **/build/test-results/test/**
             **/junit.*.xml
+            !**/binary/**
       - name: Ensure codegen is updated
         uses: ./.github/actions/ensure-codegen-updated
-      - name: Slack failure notification
-        if: failure() && github.event_name == 'push'
-        uses: kpritam/slack-job-status-action@v1
-        with:
-          job-status: ${{ job.status }}
-          slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
-          channel: github-activities
 
   quickstart-compose-validation:
     runs-on: ubuntu-latest
+    needs: setup
+    if: ${{ needs.setup.outputs.docker_change == 'true' }}
     steps:
-      - uses: actions/checkout@v3
+      - name: Check out the repo
+        uses: acryldata/sane-checkout-action@v3
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
-      - name: Download YQ
-        uses: chrisdickinson/[email protected]
-        with:
-          yq-version: v4.28.2
       - name: Quickstart Compose Validation
         run: ./docker/quickstart/generate_and_compare.sh
 

.github/workflows/check-datahub-jars.yml

@@ -27,12 +27,13 @@ jobs:
         command: ["datahub-client", "datahub-protobuf", "spark-lineage"]
     runs-on: ubuntu-latest
     steps:
-      - uses: hsheth2/sane-checkout-action@v1
-      - name: Set up JDK 11
+      - uses: acryldata/sane-checkout-action@v3
+      - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
-          java-version: 11
+          java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

.github/workflows/code-checks.yml

@@ -31,7 +31,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"